@yamo/memory-mesh 3.0.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +8 -2
  2. package/lib/llm/client.d.ts +23 -48
  3. package/lib/llm/client.js +1 -0
  4. package/lib/llm/client.ts +298 -377
  5. package/lib/llm/index.js +1 -0
  6. package/lib/llm/index.ts +1 -2
  7. package/lib/memory/adapters/client.d.ts +22 -85
  8. package/lib/memory/adapters/client.js +1 -0
  9. package/lib/memory/adapters/client.ts +474 -633
  10. package/lib/memory/adapters/config.d.ts +82 -89
  11. package/lib/memory/adapters/config.js +1 -0
  12. package/lib/memory/adapters/config.ts +156 -225
  13. package/lib/memory/adapters/errors.d.ts +28 -20
  14. package/lib/memory/adapters/errors.js +1 -0
  15. package/lib/memory/adapters/errors.ts +83 -120
  16. package/lib/memory/context-manager.d.ts +15 -18
  17. package/lib/memory/context-manager.js +1 -0
  18. package/lib/memory/context-manager.ts +314 -401
  19. package/lib/memory/embeddings/factory.d.ts +18 -20
  20. package/lib/memory/embeddings/factory.js +1 -0
  21. package/lib/memory/embeddings/factory.ts +130 -173
  22. package/lib/memory/embeddings/index.js +1 -0
  23. package/lib/memory/embeddings/index.ts +1 -0
  24. package/lib/memory/embeddings/service.d.ts +36 -66
  25. package/lib/memory/embeddings/service.js +1 -0
  26. package/lib/memory/embeddings/service.ts +479 -616
  27. package/lib/memory/index.d.ts +2 -2
  28. package/lib/memory/index.js +1 -0
  29. package/lib/memory/index.ts +3 -13
  30. package/lib/memory/memory-mesh.d.ts +151 -93
  31. package/lib/memory/memory-mesh.js +1 -0
  32. package/lib/memory/memory-mesh.ts +1406 -1692
  33. package/lib/memory/memory-translator.d.ts +1 -6
  34. package/lib/memory/memory-translator.js +1 -0
  35. package/lib/memory/memory-translator.ts +96 -128
  36. package/lib/memory/schema.d.ts +29 -10
  37. package/lib/memory/schema.js +1 -0
  38. package/lib/memory/schema.ts +102 -185
  39. package/lib/memory/scorer.d.ts +3 -4
  40. package/lib/memory/scorer.js +1 -0
  41. package/lib/memory/scorer.ts +69 -86
  42. package/lib/memory/search/index.js +1 -0
  43. package/lib/memory/search/index.ts +1 -0
  44. package/lib/memory/search/keyword-search.d.ts +10 -26
  45. package/lib/memory/search/keyword-search.js +1 -0
  46. package/lib/memory/search/keyword-search.ts +123 -161
  47. package/lib/scrubber/config/defaults.d.ts +39 -46
  48. package/lib/scrubber/config/defaults.js +1 -0
  49. package/lib/scrubber/config/defaults.ts +50 -112
  50. package/lib/scrubber/errors/scrubber-error.d.ts +22 -0
  51. package/lib/scrubber/errors/scrubber-error.js +39 -0
  52. package/lib/scrubber/errors/scrubber-error.ts +44 -0
  53. package/lib/scrubber/index.d.ts +0 -1
  54. package/lib/scrubber/index.js +1 -0
  55. package/lib/scrubber/index.ts +1 -2
  56. package/lib/scrubber/scrubber.d.ts +14 -31
  57. package/lib/scrubber/scrubber.js +1 -0
  58. package/lib/scrubber/scrubber.ts +93 -152
  59. package/lib/scrubber/stages/chunker.d.ts +22 -10
  60. package/lib/scrubber/stages/chunker.js +86 -0
  61. package/lib/scrubber/stages/chunker.ts +104 -0
  62. package/lib/scrubber/stages/metadata-annotator.d.ts +14 -15
  63. package/lib/scrubber/stages/metadata-annotator.js +64 -0
  64. package/lib/scrubber/stages/metadata-annotator.ts +75 -0
  65. package/lib/scrubber/stages/normalizer.d.ts +13 -10
  66. package/lib/scrubber/stages/normalizer.js +51 -0
  67. package/lib/scrubber/stages/normalizer.ts +60 -0
  68. package/lib/scrubber/stages/semantic-filter.d.ts +13 -10
  69. package/lib/scrubber/stages/semantic-filter.js +51 -0
  70. package/lib/scrubber/stages/semantic-filter.ts +62 -0
  71. package/lib/scrubber/stages/structural-cleaner.d.ts +15 -10
  72. package/lib/scrubber/stages/structural-cleaner.js +73 -0
  73. package/lib/scrubber/stages/structural-cleaner.ts +83 -0
  74. package/lib/scrubber/stages/validator.d.ts +14 -15
  75. package/lib/scrubber/stages/validator.js +56 -0
  76. package/lib/scrubber/stages/validator.ts +67 -0
  77. package/lib/scrubber/telemetry.d.ts +20 -27
  78. package/lib/scrubber/telemetry.js +1 -0
  79. package/lib/scrubber/telemetry.ts +53 -90
  80. package/lib/scrubber/utils/hash.d.ts +14 -0
  81. package/lib/scrubber/utils/hash.js +37 -0
  82. package/lib/scrubber/utils/hash.ts +40 -0
  83. package/lib/scrubber/utils/html-parser.d.ts +14 -0
  84. package/lib/scrubber/utils/html-parser.js +38 -0
  85. package/lib/scrubber/utils/html-parser.ts +46 -0
  86. package/lib/scrubber/utils/pattern-matcher.d.ts +12 -0
  87. package/lib/scrubber/utils/pattern-matcher.js +54 -0
  88. package/lib/scrubber/utils/pattern-matcher.ts +64 -0
  89. package/lib/scrubber/utils/token-counter.d.ts +18 -0
  90. package/lib/scrubber/utils/token-counter.js +30 -0
  91. package/lib/scrubber/utils/token-counter.ts +32 -0
  92. package/lib/utils/logger.d.ts +1 -11
  93. package/lib/utils/logger.js +1 -0
  94. package/lib/utils/logger.ts +43 -63
  95. package/lib/utils/skill-metadata.d.ts +6 -14
  96. package/lib/utils/skill-metadata.js +1 -0
  97. package/lib/utils/skill-metadata.ts +89 -103
  98. package/lib/yamo/emitter.d.ts +8 -35
  99. package/lib/yamo/emitter.js +1 -0
  100. package/lib/yamo/emitter.ts +77 -155
  101. package/lib/yamo/index.d.ts +14 -0
  102. package/lib/yamo/index.js +14 -0
  103. package/lib/yamo/index.ts +16 -0
  104. package/lib/yamo/schema.d.ts +8 -10
  105. package/lib/yamo/schema.js +1 -0
  106. package/lib/yamo/schema.ts +82 -114
  107. package/package.json +4 -2
@@ -1,3 +1,4 @@
1
+ // @ts-nocheck
1
2
  /**
2
3
  * EmbeddingService - Multi-provider embedding generation service
3
4
  *
@@ -8,646 +9,508 @@
8
9
  *
9
10
  * Implements TDD for Phase 3, Task 3.1 - Embedding Service Architecture
10
11
  */
11
-
12
12
  import crypto from "crypto";
13
13
  import { ConfigurationError, EmbeddingError } from "../adapters/errors.js";
14
-
15
- /**
16
- * Service configuration interface
17
- */
18
-
19
- export interface ServiceConfig {
20
- modelType?: "local" | "ollama" | "openai" | "cohere";
21
- modelName?: string;
22
- baseUrl?: string;
23
- dimension?: number;
24
- batchSize?: number;
25
- normalize?: boolean;
26
- cacheMaxSize?: number;
27
- apiKey?: string;
28
- priority?: number; // Used by factory
29
- }
30
-
31
- export interface ServiceStats {
32
- modelType: string;
33
- modelName: string;
34
- dimension: number;
35
- initialized: boolean;
36
- totalEmbeddings: number;
37
- cacheHits: number;
38
- cacheMisses: number;
39
- cacheSize: number;
40
- cacheMaxSize: number;
41
- cacheHitRate: number;
42
- batchCount: number;
43
- batchSize: number;
44
- normalize: boolean;
45
- }
46
-
47
14
  /**
48
15
  * EmbeddingService provides a unified interface for generating text embeddings
49
16
  * using multiple backend providers (local ONNX models or cloud APIs).
50
17
  */
51
18
  export class EmbeddingService {
52
- modelType: string;
53
- modelName: string;
54
- baseUrl: string;
55
- dimension: number;
56
- batchSize: number;
57
- normalize: boolean;
58
- apiKey?: string;
59
- model: any;
60
- cache: Map<string, number[]>;
61
- cacheMaxSize: number;
62
- initialized: boolean;
63
- stats: {
64
- totalEmbeddings: number;
65
- cacheHits: number;
66
- cacheMisses: number;
67
- batchCount: number;
68
- };
69
-
70
- /**
71
- * Create a new EmbeddingService instance
72
- * @param {Object} [config={}] - Configuration options
73
- */
74
- constructor(config: ServiceConfig = {}) {
75
- this.modelType =
76
- (config && config.modelType) ||
77
- process.env.EMBEDDING_MODEL_TYPE ||
78
- "local";
79
- this.modelName =
80
- (config && config.modelName) ||
81
- process.env.EMBEDDING_MODEL_NAME ||
82
- "Xenova/all-MiniLM-L6-v2";
83
- this.baseUrl =
84
- (config && config.baseUrl) ||
85
- process.env.OLLAMA_BASE_URL ||
86
- process.env.EMBEDDING_BASE_URL ||
87
- "http://localhost:11434";
88
- this.dimension =
89
- (config && config.dimension) ||
90
- parseInt(process.env.EMBEDDING_DIMENSION || "384") ||
91
- 384;
92
- this.batchSize =
93
- (config && config.batchSize) ||
94
- parseInt(process.env.EMBEDDING_BATCH_SIZE || "32") ||
95
- 32;
96
- this.normalize =
97
- config && config.normalize !== undefined
98
- ? config.normalize
99
- : process.env.EMBEDDING_NORMALIZE !== "false";
100
-
101
- this.apiKey = (config && config.apiKey) || process.env.EMBEDDING_API_KEY;
102
-
103
- this.model = null;
104
- this.cache = new Map();
105
- this.cacheMaxSize = (config && config.cacheMaxSize) || 1000;
106
- this.initialized = false;
107
-
108
- // Statistics
109
- this.stats = {
110
- totalEmbeddings: 0,
111
- cacheHits: 0,
112
- cacheMisses: 0,
113
- batchCount: 0,
114
- };
115
- }
116
-
117
- /**
118
- * Initialize the embedding model
119
- * Loads the model based on modelType (local, ollama, openai, cohere)
120
- */
121
- async init(): Promise<void> {
122
- try {
123
- switch (this.modelType) {
124
- case "local":
125
- await this._initLocalModel();
126
- break;
127
- case "ollama":
128
- this._initOllama();
129
- break;
130
- case "openai":
131
- await this._initOpenAI();
132
- break;
133
- case "cohere":
134
- await this._initCohere();
135
- break;
136
- default:
137
- throw new ConfigurationError(
138
- `Unknown model type: ${this.modelType}. Must be 'local', 'ollama', 'openai', or 'cohere'`,
139
- { modelType: this.modelType },
140
- );
141
- }
142
-
143
- this.initialized = true;
144
- } catch (error) {
145
- if (error instanceof ConfigurationError) {
146
- throw error;
147
- }
148
- const message = error instanceof Error ? error.message : String(error);
149
- throw new EmbeddingError(
150
- `Failed to initialize embedding service: ${message}`,
151
- {
152
- modelType: this.modelType,
153
- modelName: this.modelName,
154
- originalError: message,
155
- },
156
- );
19
+ modelType;
20
+ modelName;
21
+ baseUrl;
22
+ dimension;
23
+ batchSize;
24
+ normalize;
25
+ apiKey;
26
+ model;
27
+ cache;
28
+ cacheMaxSize;
29
+ initialized;
30
+ stats;
31
+ /**
32
+ * Create a new EmbeddingService instance
33
+ * @param {Object} [config={}] - Configuration options
34
+ */
35
+ constructor(config = {}) {
36
+ this.modelType =
37
+ (config && config.modelType) ||
38
+ process.env.EMBEDDING_MODEL_TYPE ||
39
+ "local";
40
+ this.modelName =
41
+ (config && config.modelName) ||
42
+ process.env.EMBEDDING_MODEL_NAME ||
43
+ "Xenova/all-MiniLM-L6-v2";
44
+ this.baseUrl =
45
+ (config && config.baseUrl) ||
46
+ process.env.OLLAMA_BASE_URL ||
47
+ process.env.EMBEDDING_BASE_URL ||
48
+ "http://localhost:11434";
49
+ this.dimension =
50
+ (config && config.dimension) ||
51
+ parseInt(process.env.EMBEDDING_DIMENSION || "384") ||
52
+ 384;
53
+ this.batchSize =
54
+ (config && config.batchSize) ||
55
+ parseInt(process.env.EMBEDDING_BATCH_SIZE || "32") ||
56
+ 32;
57
+ this.normalize =
58
+ config && config.normalize !== undefined
59
+ ? config.normalize
60
+ : process.env.EMBEDDING_NORMALIZE !== "false";
61
+ this.apiKey = (config && config.apiKey) || process.env.EMBEDDING_API_KEY;
62
+ this.model = null;
63
+ this.cache = new Map();
64
+ this.cacheMaxSize = (config && config.cacheMaxSize) || 1000;
65
+ this.initialized = false;
66
+ // Statistics
67
+ this.stats = {
68
+ totalEmbeddings: 0,
69
+ cacheHits: 0,
70
+ cacheMisses: 0,
71
+ batchCount: 0,
72
+ };
157
73
  }
158
- }
159
-
160
- /**
161
- * Generate embedding for a single text
162
- * @param {string} text - Text to embed
163
- * @param {Object} options - Options for embedding generation
164
- * @returns {Promise<number[]>} Embedding vector
165
- */
166
- async embed(text: string, _options: any = {}): Promise<number[]> {
167
- if (!this.initialized) {
168
- throw new EmbeddingError(
169
- "Embedding service not initialized. Call init() first.",
170
- {
171
- modelType: this.modelType,
172
- },
173
- );
74
+ /**
75
+ * Initialize the embedding model
76
+ * Loads the model based on modelType (local, ollama, openai, cohere)
77
+ */
78
+ async init() {
79
+ try {
80
+ switch (this.modelType) {
81
+ case "local":
82
+ await this._initLocalModel();
83
+ break;
84
+ case "ollama":
85
+ this._initOllama();
86
+ break;
87
+ case "openai":
88
+ await this._initOpenAI();
89
+ break;
90
+ case "cohere":
91
+ await this._initCohere();
92
+ break;
93
+ default:
94
+ throw new ConfigurationError(`Unknown model type: ${this.modelType}. Must be 'local', 'ollama', 'openai', or 'cohere'`, { modelType: this.modelType });
95
+ }
96
+ this.initialized = true;
97
+ }
98
+ catch (error) {
99
+ if (error instanceof ConfigurationError) {
100
+ throw error;
101
+ }
102
+ const message = error instanceof Error ? error.message : String(error);
103
+ throw new EmbeddingError(`Failed to initialize embedding service: ${message}`, {
104
+ modelType: this.modelType,
105
+ modelName: this.modelName,
106
+ originalError: message,
107
+ });
108
+ }
174
109
  }
175
-
176
- if (!text || typeof text !== "string") {
177
- throw new EmbeddingError("Text must be a non-empty string", {
178
- text,
179
- textType: typeof text,
180
- });
181
- }
182
-
183
- // Check cache
184
- const cacheKey = this._getCacheKey(text);
185
- const cached = this.cache.get(cacheKey);
186
- if (cached) {
187
- this.stats.cacheHits++;
188
- return cached;
189
- }
190
-
191
- // Generate embedding
192
- let embedding: number[];
193
- try {
194
- switch (this.modelType) {
195
- case "local":
196
- embedding = await this._embedLocal(text);
197
- break;
198
- case "ollama":
199
- embedding = await this._embedOllama(text);
200
- break;
201
- case "openai":
202
- embedding = await this._embedOpenAI(text);
203
- break;
204
- case "cohere":
205
- embedding = await this._embedCohere(text);
206
- break;
207
- default:
208
- throw new EmbeddingError(`Unknown model type: ${this.modelType}`, {
209
- modelType: this.modelType,
210
- });
211
- }
212
-
213
- // Normalize if enabled
214
- if (this.normalize) {
215
- embedding = this._normalize(embedding);
216
- }
217
-
218
- // Cache result
219
- this._setCache(cacheKey, embedding);
220
-
221
- this.stats.totalEmbeddings++;
222
- this.stats.cacheMisses++;
223
-
224
- return embedding;
225
- } catch (error) {
226
- if (error instanceof EmbeddingError) {
227
- throw error;
228
- }
229
- const message = error instanceof Error ? error.message : String(error);
230
- throw new EmbeddingError(`Failed to generate embedding: ${message}`, {
231
- modelType: this.modelType,
232
- text: text.substring(0, 100),
233
- });
110
+ /**
111
+ * Generate embedding for a single text
112
+ * @param {string} text - Text to embed
113
+ * @param {Object} options - Options for embedding generation
114
+ * @returns {Promise<number[]>} Embedding vector
115
+ */
116
+ async embed(text, _options = {}) {
117
+ if (!this.initialized) {
118
+ throw new EmbeddingError("Embedding service not initialized. Call init() first.", {
119
+ modelType: this.modelType,
120
+ });
121
+ }
122
+ if (!text || typeof text !== "string") {
123
+ throw new EmbeddingError("Text must be a non-empty string", {
124
+ text,
125
+ textType: typeof text,
126
+ });
127
+ }
128
+ // Check cache
129
+ const cacheKey = this._getCacheKey(text);
130
+ const cached = this.cache.get(cacheKey);
131
+ if (cached) {
132
+ this.stats.cacheHits++;
133
+ return cached;
134
+ }
135
+ // Generate embedding
136
+ let embedding;
137
+ try {
138
+ switch (this.modelType) {
139
+ case "local":
140
+ embedding = await this._embedLocal(text);
141
+ break;
142
+ case "ollama":
143
+ embedding = await this._embedOllama(text);
144
+ break;
145
+ case "openai":
146
+ embedding = await this._embedOpenAI(text);
147
+ break;
148
+ case "cohere":
149
+ embedding = await this._embedCohere(text);
150
+ break;
151
+ default:
152
+ throw new EmbeddingError(`Unknown model type: ${this.modelType}`, {
153
+ modelType: this.modelType,
154
+ });
155
+ }
156
+ // Normalize if enabled
157
+ if (this.normalize) {
158
+ embedding = this._normalize(embedding);
159
+ }
160
+ // Cache result
161
+ this._setCache(cacheKey, embedding);
162
+ this.stats.totalEmbeddings++;
163
+ this.stats.cacheMisses++;
164
+ return embedding;
165
+ }
166
+ catch (error) {
167
+ if (error instanceof EmbeddingError) {
168
+ throw error;
169
+ }
170
+ const message = error instanceof Error ? error.message : String(error);
171
+ throw new EmbeddingError(`Failed to generate embedding: ${message}`, {
172
+ modelType: this.modelType,
173
+ text: text.substring(0, 100),
174
+ });
175
+ }
234
176
  }
235
- }
236
-
237
- /**
238
- * Generate embeddings for a batch of texts
239
- * @param {string[]} texts - Array of texts to embed
240
- * @param {Object} options - Options for embedding generation
241
- * @returns {Promise<number[][]>} Array of embedding vectors
242
- */
243
- async embedBatch(texts: string[], _options: any = {}): Promise<number[][]> {
244
- if (!this.initialized) {
245
- throw new EmbeddingError(
246
- "Embedding service not initialized. Call init() first.",
247
- {
248
- modelType: this.modelType,
249
- },
250
- );
177
+ /**
178
+ * Generate embeddings for a batch of texts
179
+ * @param {string[]} texts - Array of texts to embed
180
+ * @param {Object} options - Options for embedding generation
181
+ * @returns {Promise<number[][]>} Array of embedding vectors
182
+ */
183
+ async embedBatch(texts, _options = {}) {
184
+ if (!this.initialized) {
185
+ throw new EmbeddingError("Embedding service not initialized. Call init() first.", {
186
+ modelType: this.modelType,
187
+ });
188
+ }
189
+ if (!Array.isArray(texts)) {
190
+ throw new EmbeddingError("Texts must be an array", {
191
+ textsType: typeof texts,
192
+ });
193
+ }
194
+ if (texts.length === 0) {
195
+ return [];
196
+ }
197
+ try {
198
+ const embeddings = [];
199
+ // Process in batches
200
+ for (let i = 0; i < texts.length; i += this.batchSize) {
201
+ const batch = texts.slice(i, Math.min(i + this.batchSize, texts.length));
202
+ // Generate embeddings for batch
203
+ const batchEmbeddings = await Promise.all(batch.map((text) => this.embed(text)));
204
+ embeddings.push(...batchEmbeddings);
205
+ this.stats.batchCount++;
206
+ }
207
+ return embeddings;
208
+ }
209
+ catch (error) {
210
+ if (error instanceof EmbeddingError) {
211
+ throw error;
212
+ }
213
+ const message = error instanceof Error ? error.message : String(error);
214
+ throw new EmbeddingError(`Failed to generate batch embeddings: ${message}`, {
215
+ modelType: this.modelType,
216
+ batchSize: texts.length,
217
+ });
218
+ }
251
219
  }
252
-
253
- if (!Array.isArray(texts)) {
254
- throw new EmbeddingError("Texts must be an array", {
255
- textsType: typeof texts,
256
- });
220
+ /**
221
+ * Initialize local ONNX model using Xenova/Transformers.js
222
+ * @private
223
+ */
224
+ async _initLocalModel() {
225
+ try {
226
+ // Dynamic import to allow optional dependency
227
+ const { pipeline } = (await import("@xenova/transformers"));
228
+ // Load feature extraction pipeline
229
+ this.model = await pipeline("feature-extraction", this.modelName, {
230
+ quantized: true,
231
+ progress_callback: (progress) => {
232
+ // Optional: Log model download progress
233
+ if (progress.status === "downloading") {
234
+ // Silently handle progress
235
+ }
236
+ },
237
+ });
238
+ // Update dimension based on model (384 for all-MiniLM-L6-v2)
239
+ if (this.modelName.includes("all-MiniLM-L6-v2")) {
240
+ this.dimension = 384;
241
+ }
242
+ }
243
+ catch (error) {
244
+ const message = error instanceof Error ? error.message : String(error);
245
+ throw new ConfigurationError(`Failed to load local model: ${message}. Make sure @xenova/transformers is installed.`, { modelName: this.modelName, error: message });
246
+ }
257
247
  }
258
-
259
- if (texts.length === 0) {
260
- return [];
248
+ /**
249
+ * Initialize Ollama client
250
+ * Ollama runs locally and doesn't require authentication
251
+ * @private
252
+ */
253
+ _initOllama() {
254
+ // Ollama doesn't require initialization - it's a local HTTP API
255
+ // Store the base URL for use in _embedOllama
256
+ this.model = {
257
+ baseUrl: this.baseUrl,
258
+ modelName: this.modelName || "nomic-embed-text",
259
+ };
260
+ // Set default dimension for common Ollama embedding models
261
+ if (this.modelName.includes("nomic-embed-text")) {
262
+ this.dimension = 768;
263
+ }
264
+ else if (this.modelName.includes("mxbai-embed")) {
265
+ this.dimension = 1024;
266
+ }
267
+ else if (this.modelName.includes("all-MiniLM")) {
268
+ this.dimension = 384;
269
+ }
261
270
  }
262
-
263
- try {
264
- const embeddings: number[][] = [];
265
-
266
- // Process in batches
267
- for (let i = 0; i < texts.length; i += this.batchSize) {
268
- const batch = texts.slice(
269
- i,
270
- Math.min(i + this.batchSize, texts.length),
271
- );
272
-
273
- // Generate embeddings for batch
274
- const batchEmbeddings = await Promise.all(
275
- batch.map((text) => this.embed(text)),
276
- );
277
-
278
- embeddings.push(...batchEmbeddings);
279
- this.stats.batchCount++;
280
- }
281
-
282
- return embeddings;
283
- } catch (error) {
284
- if (error instanceof EmbeddingError) {
285
- throw error;
286
- }
287
- const message = error instanceof Error ? error.message : String(error);
288
- throw new EmbeddingError(
289
- `Failed to generate batch embeddings: ${message}`,
290
- {
291
- modelType: this.modelType,
292
- batchSize: texts.length,
293
- },
294
- );
271
+ /**
272
+ * Initialize OpenAI client
273
+ * @private
274
+ */
275
+ async _initOpenAI() {
276
+ if (!this.apiKey) {
277
+ throw new ConfigurationError("OpenAI API key is required. Set EMBEDDING_API_KEY environment variable or pass apiKey in config.", { modelType: "openai" });
278
+ }
279
+ try {
280
+ // Dynamic import to allow optional dependency (openai may not be installed)
281
+ const { OpenAI } = await import("openai");
282
+ this.model = new OpenAI({ apiKey: this.apiKey });
283
+ // Update dimension for OpenAI models
284
+ if (this.modelName.includes("text-embedding-ada-002")) {
285
+ this.dimension = 1536;
286
+ }
287
+ }
288
+ catch (error) {
289
+ const message = error instanceof Error ? error.message : String(error);
290
+ throw new ConfigurationError(`Failed to initialize OpenAI client: ${message}. Make sure openai package is installed.`, { error: message });
291
+ }
295
292
  }
296
- }
297
-
298
- /**
299
- * Initialize local ONNX model using Xenova/Transformers.js
300
- * @private
301
- */
302
- async _initLocalModel(): Promise<void> {
303
- try {
304
- // Dynamic import to allow optional dependency
305
- const { pipeline } = (await import("@xenova/transformers")) as any;
306
-
307
- // Load feature extraction pipeline
308
- this.model = await pipeline("feature-extraction", this.modelName, {
309
- quantized: true,
310
- progress_callback: (progress: any) => {
311
- // Optional: Log model download progress
312
- if (progress.status === "downloading") {
313
- // Silently handle progress
314
- }
315
- },
316
- });
317
-
318
- // Update dimension based on model (384 for all-MiniLM-L6-v2)
319
- if (this.modelName.includes("all-MiniLM-L6-v2")) {
320
- this.dimension = 384;
321
- }
322
- } catch (error) {
323
- const message = error instanceof Error ? error.message : String(error);
324
- throw new ConfigurationError(
325
- `Failed to load local model: ${message}. Make sure @xenova/transformers is installed.`,
326
- { modelName: this.modelName, error: message },
327
- );
293
+ /**
294
+ * Initialize Cohere client
295
+ * @private
296
+ */
297
+ async _initCohere() {
298
+ if (!this.apiKey) {
299
+ throw new ConfigurationError("Cohere API key is required. Set EMBEDDING_API_KEY environment variable or pass apiKey in config.", { modelType: "cohere" });
300
+ }
301
+ try {
302
+ // Dynamic import to allow optional dependency (cohere-ai may not be installed)
303
+ const cohere = await import("cohere-ai");
304
+ this.model = new cohere.CohereClient({ token: this.apiKey });
305
+ // Update dimension for Cohere models
306
+ if (this.modelName.includes("embed-english-v3.0")) {
307
+ this.dimension = 1024;
308
+ }
309
+ }
310
+ catch (error) {
311
+ const message = error instanceof Error ? error.message : String(error);
312
+ throw new ConfigurationError(`Failed to initialize Cohere client: ${message}. Make sure cohere-ai package is installed.`, { error: message });
313
+ }
328
314
  }
329
- }
330
-
331
- /**
332
- * Initialize Ollama client
333
- * Ollama runs locally and doesn't require authentication
334
- * @private
335
- */
336
- _initOllama(): void {
337
- // Ollama doesn't require initialization - it's a local HTTP API
338
- // Store the base URL for use in _embedOllama
339
- this.model = {
340
- baseUrl: this.baseUrl,
341
- modelName: this.modelName || "nomic-embed-text",
342
- };
343
-
344
- // Set default dimension for common Ollama embedding models
345
- if (this.modelName.includes("nomic-embed-text")) {
346
- this.dimension = 768;
347
- } else if (this.modelName.includes("mxbai-embed")) {
348
- this.dimension = 1024;
349
- } else if (this.modelName.includes("all-MiniLM")) {
350
- this.dimension = 384;
315
+ /**
316
+ * Generate embedding using local ONNX model
317
+ * @param {string} text - Text to embed
318
+ * @returns {Promise<number[]>} Embedding vector
319
+ * @private
320
+ */
321
+ async _embedLocal(text) {
322
+ if (!this.model) {
323
+ throw new EmbeddingError("Model not initialized");
324
+ }
325
+ try {
326
+ // Local model call
327
+ const output = await this.model(text, {
328
+ pooling: "mean",
329
+ normalize: false,
330
+ });
331
+ // Convert from tensor to array
332
+ const embedding = Array.from(output.data);
333
+ return embedding;
334
+ }
335
+ catch (error) {
336
+ const message = error instanceof Error ? error.message : String(error);
337
+ throw new EmbeddingError(`Failed to generate local embedding: ${message}`, {
338
+ modelName: this.modelName,
339
+ text: text.substring(0, 100),
340
+ });
341
+ }
351
342
  }
352
- }
353
-
354
- /**
355
- * Initialize OpenAI client
356
- * @private
357
- */
358
- async _initOpenAI(): Promise<void> {
359
- if (!this.apiKey) {
360
- throw new ConfigurationError(
361
- "OpenAI API key is required. Set EMBEDDING_API_KEY environment variable or pass apiKey in config.",
362
- { modelType: "openai" },
363
- );
343
+ /**
344
+ * Generate embedding using Ollama API
345
+ * @param {string} text - Text to embed
346
+ * @returns {Promise<number[]>} Embedding vector
347
+ * @private
348
+ */
349
+ async _embedOllama(text) {
350
+ if (!this.model) {
351
+ throw new EmbeddingError("Model not initialized");
352
+ }
353
+ try {
354
+ const baseUrl = this.model.baseUrl;
355
+ const modelName = this.model.modelName;
356
+ const response = await fetch(`${baseUrl}/api/embeddings`, {
357
+ method: "POST",
358
+ headers: {
359
+ "Content-Type": "application/json",
360
+ },
361
+ body: JSON.stringify({
362
+ model: modelName,
363
+ prompt: text,
364
+ }),
365
+ });
366
+ if (!response.ok) {
367
+ const errorText = await response.text();
368
+ throw new EmbeddingError(`Ollama API error: ${response.status} ${response.statusText} - ${errorText}`, { baseUrl: baseUrl, modelName: modelName });
369
+ }
370
+ const data = await response.json();
371
+ if (!data.embedding) {
372
+ throw new EmbeddingError("Invalid response from Ollama API: missing embedding field", {
373
+ response: data,
374
+ });
375
+ }
376
+ return data.embedding;
377
+ }
378
+ catch (error) {
379
+ if (error instanceof EmbeddingError) {
380
+ throw error;
381
+ }
382
+ const message = error instanceof Error ? error.message : String(error);
383
+ const baseUrl = this.model?.baseUrl;
384
+ const modelName = this.model?.modelName;
385
+ throw new EmbeddingError(`Failed to generate Ollama embedding: ${message}. Make sure Ollama is running and the model is available.`, { baseUrl, modelName, error: message });
386
+ }
364
387
  }
365
-
366
- try {
367
- // Dynamic import to allow optional dependency (openai may not be installed)
368
- const { OpenAI } = await import("openai" as any);
369
- this.model = new OpenAI({ apiKey: this.apiKey });
370
-
371
- // Update dimension for OpenAI models
372
- if (this.modelName.includes("text-embedding-ada-002")) {
373
- this.dimension = 1536;
374
- }
375
- } catch (error) {
376
- const message = error instanceof Error ? error.message : String(error);
377
- throw new ConfigurationError(
378
- `Failed to initialize OpenAI client: ${message}. Make sure openai package is installed.`,
379
- { error: message },
380
- );
388
+ /**
389
+ * Generate embedding using OpenAI API
390
+ * @param {string} text - Text to embed
391
+ * @returns {Promise<number[]>} Embedding vector
392
+ * @private
393
+ */
394
+ async _embedOpenAI(text) {
395
+ if (!this.model) {
396
+ throw new EmbeddingError("Model not initialized");
397
+ }
398
+ try {
399
+ const response = await this.model.embeddings.create({
400
+ model: this.modelName,
401
+ input: text,
402
+ });
403
+ const embedding = response.data[0].embedding;
404
+ return embedding;
405
+ }
406
+ catch (error) {
407
+ const message = error instanceof Error ? error.message : String(error);
408
+ throw new EmbeddingError(`Failed to generate OpenAI embedding: ${message}`, {
409
+ modelName: this.modelName,
410
+ error: message,
411
+ });
412
+ }
381
413
  }
382
- }
383
-
384
- /**
385
- * Initialize Cohere client
386
- * @private
387
- */
388
- async _initCohere(): Promise<void> {
389
- if (!this.apiKey) {
390
- throw new ConfigurationError(
391
- "Cohere API key is required. Set EMBEDDING_API_KEY environment variable or pass apiKey in config.",
392
- { modelType: "cohere" },
393
- );
414
+ /**
415
+ * Generate embedding using Cohere API
416
+ * @param {string} text - Text to embed
417
+ * @returns {Promise<number[]>} Embedding vector
418
+ * @private
419
+ */
420
+ async _embedCohere(text) {
421
+ if (!this.model) {
422
+ throw new EmbeddingError("Model not initialized");
423
+ }
424
+ try {
425
+ const response = await this.model.embed({
426
+ model: this.modelName,
427
+ texts: [text],
428
+ inputType: "search_document",
429
+ });
430
+ const embedding = response.embeddings[0];
431
+ return embedding;
432
+ }
433
+ catch (error) {
434
+ const message = error instanceof Error ? error.message : String(error);
435
+ throw new EmbeddingError(`Failed to generate Cohere embedding: ${message}`, {
436
+ modelName: this.modelName,
437
+ error: message,
438
+ });
439
+ }
394
440
  }
395
-
396
- try {
397
- // Dynamic import to allow optional dependency (cohere-ai may not be installed)
398
- const cohere = await import("cohere-ai" as any);
399
- this.model = new cohere.CohereClient({ token: this.apiKey });
400
-
401
- // Update dimension for Cohere models
402
- if (this.modelName.includes("embed-english-v3.0")) {
403
- this.dimension = 1024;
404
- }
405
- } catch (error) {
406
- const message = error instanceof Error ? error.message : String(error);
407
- throw new ConfigurationError(
408
- `Failed to initialize Cohere client: ${message}. Make sure cohere-ai package is installed.`,
409
- { error: message },
410
- );
441
+ /**
442
+ * Normalize vector to unit length
443
+ * @param {number[]} vector - Vector to normalize
444
+ * @returns {number[]} Normalized vector
445
+ * @private
446
+ */
447
+ _normalize(vector) {
448
+ // Calculate magnitude
449
+ const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
450
+ // Avoid division by zero
451
+ if (magnitude === 0) {
452
+ return vector.map(() => 0);
453
+ }
454
+ // Normalize
455
+ return vector.map((val) => val / magnitude);
411
456
  }
412
- }
413
-
414
- /**
415
- * Generate embedding using local ONNX model
416
- * @param {string} text - Text to embed
417
- * @returns {Promise<number[]>} Embedding vector
418
- * @private
419
- */
420
- async _embedLocal(text: string): Promise<number[]> {
421
- if (!this.model) {
422
- throw new EmbeddingError("Model not initialized");
457
+ /**
458
+ * Generate cache key from text
459
+ * @param {string} text - Text to generate key from
460
+ * @returns {string} Cache key
461
+ * @private
462
+ */
463
+ _getCacheKey(text) {
464
+ return crypto.createHash("md5").update(text).digest("hex");
423
465
  }
424
- try {
425
- // Local model call
426
- const output = await this.model(text, {
427
- pooling: "mean",
428
- normalize: false,
429
- });
430
-
431
- // Convert from tensor to array
432
- const embedding = Array.from(output.data);
433
- return embedding as number[];
434
- } catch (error) {
435
- const message = error instanceof Error ? error.message : String(error);
436
- throw new EmbeddingError(
437
- `Failed to generate local embedding: ${message}`,
438
- {
439
- modelName: this.modelName,
440
- text: text.substring(0, 100),
441
- },
442
- );
466
+ _setCache(key, value) {
467
+ // Evict oldest if at capacity
468
+ if (this.cache.size >= this.cacheMaxSize) {
469
+ const firstKey = this.cache.keys().next().value;
470
+ if (firstKey !== undefined) {
471
+ this.cache.delete(firstKey);
472
+ }
473
+ }
474
+ this.cache.set(key, value);
443
475
  }
444
- }
445
-
446
- /**
447
- * Generate embedding using Ollama API
448
- * @param {string} text - Text to embed
449
- * @returns {Promise<number[]>} Embedding vector
450
- * @private
451
- */
452
- async _embedOllama(text: string): Promise<number[]> {
453
- if (!this.model) {
454
- throw new EmbeddingError("Model not initialized");
455
- }
456
- try {
457
- const baseUrl = this.model.baseUrl;
458
- const modelName = this.model.modelName;
459
-
460
- const response = await fetch(`${baseUrl}/api/embeddings`, {
461
- method: "POST",
462
- headers: {
463
- "Content-Type": "application/json",
464
- },
465
- body: JSON.stringify({
466
- model: modelName,
467
- prompt: text,
468
- }),
469
- });
470
-
471
- if (!response.ok) {
472
- const errorText = await response.text();
473
- throw new EmbeddingError(
474
- `Ollama API error: ${response.status} ${response.statusText} - ${errorText}`,
475
- { baseUrl: baseUrl, modelName: modelName },
476
- );
477
- }
478
-
479
- const data = await response.json();
480
-
481
- if (!data.embedding) {
482
- throw new EmbeddingError(
483
- "Invalid response from Ollama API: missing embedding field",
484
- {
485
- response: data,
486
- },
487
- );
488
- }
489
-
490
- return data.embedding;
491
- } catch (error) {
492
- if (error instanceof EmbeddingError) {
493
- throw error;
494
- }
495
- const message = error instanceof Error ? error.message : String(error);
496
- const baseUrl = this.model?.baseUrl;
497
- const modelName = this.model?.modelName;
498
- throw new EmbeddingError(
499
- `Failed to generate Ollama embedding: ${message}. Make sure Ollama is running and the model is available.`,
500
- { baseUrl, modelName, error: message },
501
- );
502
- }
503
- }
504
-
505
- /**
506
- * Generate embedding using OpenAI API
507
- * @param {string} text - Text to embed
508
- * @returns {Promise<number[]>} Embedding vector
509
- * @private
510
- */
511
- async _embedOpenAI(text: string): Promise<number[]> {
512
- if (!this.model) {
513
- throw new EmbeddingError("Model not initialized");
514
- }
515
- try {
516
- const response = await this.model.embeddings.create({
517
- model: this.modelName,
518
- input: text,
519
- });
520
-
521
- const embedding = response.data[0].embedding;
522
- return embedding;
523
- } catch (error) {
524
- const message = error instanceof Error ? error.message : String(error);
525
- throw new EmbeddingError(
526
- `Failed to generate OpenAI embedding: ${message}`,
527
- {
528
- modelName: this.modelName,
529
- error: message,
530
- },
531
- );
532
- }
533
- }
534
-
535
- /**
536
- * Generate embedding using Cohere API
537
- * @param {string} text - Text to embed
538
- * @returns {Promise<number[]>} Embedding vector
539
- * @private
540
- */
541
- async _embedCohere(text: string): Promise<number[]> {
542
- if (!this.model) {
543
- throw new EmbeddingError("Model not initialized");
544
- }
545
- try {
546
- const response = await this.model.embed({
547
- model: this.modelName,
548
- texts: [text],
549
- inputType: "search_document",
550
- });
551
-
552
- const embedding = response.embeddings[0];
553
- return embedding;
554
- } catch (error) {
555
- const message = error instanceof Error ? error.message : String(error);
556
- throw new EmbeddingError(
557
- `Failed to generate Cohere embedding: ${message}`,
558
- {
559
- modelName: this.modelName,
560
- error: message,
561
- },
562
- );
476
+ /**
477
+ * Get service statistics
478
+ * @returns {Object} Statistics object
479
+ */
480
+ getStats() {
481
+ return {
482
+ modelType: this.modelType,
483
+ modelName: this.modelName,
484
+ dimension: this.dimension,
485
+ initialized: this.initialized,
486
+ totalEmbeddings: this.stats.totalEmbeddings,
487
+ cacheHits: this.stats.cacheHits,
488
+ cacheMisses: this.stats.cacheMisses,
489
+ cacheSize: this.cache.size,
490
+ cacheMaxSize: this.cacheMaxSize,
491
+ cacheHitRate: this.stats.cacheHits /
492
+ (this.stats.cacheHits + this.stats.cacheMisses) || 0,
493
+ batchCount: this.stats.batchCount,
494
+ batchSize: this.batchSize,
495
+ normalize: this.normalize,
496
+ };
563
497
  }
564
- }
565
-
566
- /**
567
- * Normalize vector to unit length
568
- * @param {number[]} vector - Vector to normalize
569
- * @returns {number[]} Normalized vector
570
- * @private
571
- */
572
- _normalize(vector: number[]): number[] {
573
- // Calculate magnitude
574
- const magnitude = Math.sqrt(
575
- vector.reduce((sum, val) => sum + val * val, 0),
576
- );
577
-
578
- // Avoid division by zero
579
- if (magnitude === 0) {
580
- return vector.map(() => 0);
498
+ /**
499
+ * Clear the embedding cache
500
+ */
501
+ clearCache() {
502
+ this.cache.clear();
581
503
  }
582
-
583
- // Normalize
584
- return vector.map((val) => val / magnitude);
585
- }
586
-
587
- /**
588
- * Generate cache key from text
589
- * @param {string} text - Text to generate key from
590
- * @returns {string} Cache key
591
- * @private
592
- */
593
- _getCacheKey(text: string): string {
594
- return crypto.createHash("md5").update(text).digest("hex");
595
- }
596
-
597
- _setCache(key: string, value: number[]): void {
598
- // Evict oldest if at capacity
599
- if (this.cache.size >= this.cacheMaxSize) {
600
- const firstKey = this.cache.keys().next().value;
601
- if (firstKey !== undefined) {
602
- this.cache.delete(firstKey);
603
- }
504
+ /**
505
+ * Reset statistics
506
+ */
507
+ resetStats() {
508
+ this.stats = {
509
+ totalEmbeddings: 0,
510
+ cacheHits: 0,
511
+ cacheMisses: 0,
512
+ batchCount: 0,
513
+ };
604
514
  }
605
-
606
- this.cache.set(key, value);
607
- }
608
-
609
- /**
610
- * Get service statistics
611
- * @returns {Object} Statistics object
612
- */
613
- getStats(): ServiceStats {
614
- return {
615
- modelType: this.modelType,
616
- modelName: this.modelName,
617
- dimension: this.dimension,
618
- initialized: this.initialized,
619
- totalEmbeddings: this.stats.totalEmbeddings,
620
- cacheHits: this.stats.cacheHits,
621
- cacheMisses: this.stats.cacheMisses,
622
- cacheSize: this.cache.size,
623
- cacheMaxSize: this.cacheMaxSize,
624
- cacheHitRate:
625
- this.stats.cacheHits /
626
- (this.stats.cacheHits + this.stats.cacheMisses) || 0,
627
- batchCount: this.stats.batchCount,
628
- batchSize: this.batchSize,
629
- normalize: this.normalize,
630
- };
631
- }
632
-
633
- /**
634
- * Clear the embedding cache
635
- */
636
- clearCache(): void {
637
- this.cache.clear();
638
- }
639
-
640
- /**
641
- * Reset statistics
642
- */
643
- resetStats(): void {
644
- this.stats = {
645
- totalEmbeddings: 0,
646
- cacheHits: 0,
647
- cacheMisses: 0,
648
- batchCount: 0,
649
- };
650
- }
651
515
  }
652
-
653
516
  export default EmbeddingService;