@danielsimonjr/memoryjs 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +266 -0
  3. package/dist/core/EntityManager.d.ts +268 -0
  4. package/dist/core/EntityManager.d.ts.map +1 -0
  5. package/dist/core/EntityManager.js +512 -0
  6. package/dist/core/EntityManager.js.map +1 -0
  7. package/dist/core/GraphEventEmitter.d.ts +202 -0
  8. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  9. package/dist/core/GraphEventEmitter.js +347 -0
  10. package/dist/core/GraphEventEmitter.js.map +1 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +786 -0
  14. package/dist/core/GraphStorage.js.map +1 -0
  15. package/dist/core/GraphTraversal.d.ts +141 -0
  16. package/dist/core/GraphTraversal.d.ts.map +1 -0
  17. package/dist/core/GraphTraversal.js +574 -0
  18. package/dist/core/GraphTraversal.js.map +1 -0
  19. package/dist/core/HierarchyManager.d.ts +111 -0
  20. package/dist/core/HierarchyManager.d.ts.map +1 -0
  21. package/dist/core/HierarchyManager.js +225 -0
  22. package/dist/core/HierarchyManager.js.map +1 -0
  23. package/dist/core/ManagerContext.d.ts +76 -0
  24. package/dist/core/ManagerContext.d.ts.map +1 -0
  25. package/dist/core/ManagerContext.js +129 -0
  26. package/dist/core/ManagerContext.js.map +1 -0
  27. package/dist/core/ObservationManager.d.ts +85 -0
  28. package/dist/core/ObservationManager.d.ts.map +1 -0
  29. package/dist/core/ObservationManager.js +124 -0
  30. package/dist/core/ObservationManager.js.map +1 -0
  31. package/dist/core/RelationManager.d.ts +131 -0
  32. package/dist/core/RelationManager.d.ts.map +1 -0
  33. package/dist/core/RelationManager.js +212 -0
  34. package/dist/core/RelationManager.js.map +1 -0
  35. package/dist/core/SQLiteStorage.d.ts +354 -0
  36. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  37. package/dist/core/SQLiteStorage.js +919 -0
  38. package/dist/core/SQLiteStorage.js.map +1 -0
  39. package/dist/core/StorageFactory.d.ts +45 -0
  40. package/dist/core/StorageFactory.d.ts.map +1 -0
  41. package/dist/core/StorageFactory.js +65 -0
  42. package/dist/core/StorageFactory.js.map +1 -0
  43. package/dist/core/TransactionManager.d.ts +464 -0
  44. package/dist/core/TransactionManager.d.ts.map +1 -0
  45. package/dist/core/TransactionManager.js +869 -0
  46. package/dist/core/TransactionManager.js.map +1 -0
  47. package/dist/core/index.d.ts +17 -0
  48. package/dist/core/index.d.ts.map +1 -0
  49. package/dist/core/index.js +20 -0
  50. package/dist/core/index.js.map +1 -0
  51. package/dist/features/AnalyticsManager.d.ts +44 -0
  52. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  53. package/dist/features/AnalyticsManager.js +224 -0
  54. package/dist/features/AnalyticsManager.js.map +1 -0
  55. package/dist/features/ArchiveManager.d.ts +133 -0
  56. package/dist/features/ArchiveManager.d.ts.map +1 -0
  57. package/dist/features/ArchiveManager.js +282 -0
  58. package/dist/features/ArchiveManager.js.map +1 -0
  59. package/dist/features/CompressionManager.d.ts +119 -0
  60. package/dist/features/CompressionManager.d.ts.map +1 -0
  61. package/dist/features/CompressionManager.js +470 -0
  62. package/dist/features/CompressionManager.js.map +1 -0
  63. package/dist/features/IOManager.d.ts +225 -0
  64. package/dist/features/IOManager.d.ts.map +1 -0
  65. package/dist/features/IOManager.js +1093 -0
  66. package/dist/features/IOManager.js.map +1 -0
  67. package/dist/features/KeywordExtractor.d.ts +61 -0
  68. package/dist/features/KeywordExtractor.d.ts.map +1 -0
  69. package/dist/features/KeywordExtractor.js +127 -0
  70. package/dist/features/KeywordExtractor.js.map +1 -0
  71. package/dist/features/ObservationNormalizer.d.ts +90 -0
  72. package/dist/features/ObservationNormalizer.d.ts.map +1 -0
  73. package/dist/features/ObservationNormalizer.js +194 -0
  74. package/dist/features/ObservationNormalizer.js.map +1 -0
  75. package/dist/features/StreamingExporter.d.ts +128 -0
  76. package/dist/features/StreamingExporter.d.ts.map +1 -0
  77. package/dist/features/StreamingExporter.js +212 -0
  78. package/dist/features/StreamingExporter.js.map +1 -0
  79. package/dist/features/TagManager.d.ts +147 -0
  80. package/dist/features/TagManager.d.ts.map +1 -0
  81. package/dist/features/TagManager.js +211 -0
  82. package/dist/features/TagManager.js.map +1 -0
  83. package/dist/features/index.d.ts +14 -0
  84. package/dist/features/index.d.ts.map +1 -0
  85. package/dist/features/index.js +15 -0
  86. package/dist/features/index.js.map +1 -0
  87. package/dist/index.d.ts +15 -0
  88. package/dist/index.d.ts.map +1 -0
  89. package/dist/index.js +20 -0
  90. package/dist/index.js.map +1 -0
  91. package/dist/search/BM25Search.d.ts +148 -0
  92. package/dist/search/BM25Search.d.ts.map +1 -0
  93. package/dist/search/BM25Search.js +340 -0
  94. package/dist/search/BM25Search.js.map +1 -0
  95. package/dist/search/BasicSearch.d.ts +51 -0
  96. package/dist/search/BasicSearch.d.ts.map +1 -0
  97. package/dist/search/BasicSearch.js +138 -0
  98. package/dist/search/BasicSearch.js.map +1 -0
  99. package/dist/search/BooleanSearch.d.ts +98 -0
  100. package/dist/search/BooleanSearch.d.ts.map +1 -0
  101. package/dist/search/BooleanSearch.js +431 -0
  102. package/dist/search/BooleanSearch.js.map +1 -0
  103. package/dist/search/EarlyTerminationManager.d.ts +140 -0
  104. package/dist/search/EarlyTerminationManager.d.ts.map +1 -0
  105. package/dist/search/EarlyTerminationManager.js +280 -0
  106. package/dist/search/EarlyTerminationManager.js.map +1 -0
  107. package/dist/search/EmbeddingCache.d.ts +175 -0
  108. package/dist/search/EmbeddingCache.d.ts.map +1 -0
  109. package/dist/search/EmbeddingCache.js +247 -0
  110. package/dist/search/EmbeddingCache.js.map +1 -0
  111. package/dist/search/EmbeddingService.d.ts +277 -0
  112. package/dist/search/EmbeddingService.d.ts.map +1 -0
  113. package/dist/search/EmbeddingService.js +531 -0
  114. package/dist/search/EmbeddingService.js.map +1 -0
  115. package/dist/search/FuzzySearch.d.ts +118 -0
  116. package/dist/search/FuzzySearch.d.ts.map +1 -0
  117. package/dist/search/FuzzySearch.js +313 -0
  118. package/dist/search/FuzzySearch.js.map +1 -0
  119. package/dist/search/HybridScorer.d.ts +181 -0
  120. package/dist/search/HybridScorer.d.ts.map +1 -0
  121. package/dist/search/HybridScorer.js +258 -0
  122. package/dist/search/HybridScorer.js.map +1 -0
  123. package/dist/search/HybridSearchManager.d.ts +80 -0
  124. package/dist/search/HybridSearchManager.d.ts.map +1 -0
  125. package/dist/search/HybridSearchManager.js +188 -0
  126. package/dist/search/HybridSearchManager.js.map +1 -0
  127. package/dist/search/IncrementalIndexer.d.ts +201 -0
  128. package/dist/search/IncrementalIndexer.d.ts.map +1 -0
  129. package/dist/search/IncrementalIndexer.js +343 -0
  130. package/dist/search/IncrementalIndexer.js.map +1 -0
  131. package/dist/search/OptimizedInvertedIndex.d.ts +163 -0
  132. package/dist/search/OptimizedInvertedIndex.d.ts.map +1 -0
  133. package/dist/search/OptimizedInvertedIndex.js +359 -0
  134. package/dist/search/OptimizedInvertedIndex.js.map +1 -0
  135. package/dist/search/ParallelSearchExecutor.d.ts +172 -0
  136. package/dist/search/ParallelSearchExecutor.d.ts.map +1 -0
  137. package/dist/search/ParallelSearchExecutor.js +310 -0
  138. package/dist/search/ParallelSearchExecutor.js.map +1 -0
  139. package/dist/search/QuantizedVectorStore.d.ts +171 -0
  140. package/dist/search/QuantizedVectorStore.d.ts.map +1 -0
  141. package/dist/search/QuantizedVectorStore.js +308 -0
  142. package/dist/search/QuantizedVectorStore.js.map +1 -0
  143. package/dist/search/QueryAnalyzer.d.ts +76 -0
  144. package/dist/search/QueryAnalyzer.d.ts.map +1 -0
  145. package/dist/search/QueryAnalyzer.js +228 -0
  146. package/dist/search/QueryAnalyzer.js.map +1 -0
  147. package/dist/search/QueryCostEstimator.d.ts +244 -0
  148. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  149. package/dist/search/QueryCostEstimator.js +653 -0
  150. package/dist/search/QueryCostEstimator.js.map +1 -0
  151. package/dist/search/QueryPlanCache.d.ts +220 -0
  152. package/dist/search/QueryPlanCache.d.ts.map +1 -0
  153. package/dist/search/QueryPlanCache.js +380 -0
  154. package/dist/search/QueryPlanCache.js.map +1 -0
  155. package/dist/search/QueryPlanner.d.ts +58 -0
  156. package/dist/search/QueryPlanner.d.ts.map +1 -0
  157. package/dist/search/QueryPlanner.js +138 -0
  158. package/dist/search/QueryPlanner.js.map +1 -0
  159. package/dist/search/RankedSearch.d.ts +71 -0
  160. package/dist/search/RankedSearch.d.ts.map +1 -0
  161. package/dist/search/RankedSearch.js +239 -0
  162. package/dist/search/RankedSearch.js.map +1 -0
  163. package/dist/search/ReflectionManager.d.ts +120 -0
  164. package/dist/search/ReflectionManager.d.ts.map +1 -0
  165. package/dist/search/ReflectionManager.js +232 -0
  166. package/dist/search/ReflectionManager.js.map +1 -0
  167. package/dist/search/SavedSearchManager.d.ts +79 -0
  168. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  169. package/dist/search/SavedSearchManager.js +147 -0
  170. package/dist/search/SavedSearchManager.js.map +1 -0
  171. package/dist/search/SearchFilterChain.d.ts +120 -0
  172. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  173. package/dist/search/SearchFilterChain.js +186 -0
  174. package/dist/search/SearchFilterChain.js.map +1 -0
  175. package/dist/search/SearchManager.d.ts +326 -0
  176. package/dist/search/SearchManager.d.ts.map +1 -0
  177. package/dist/search/SearchManager.js +454 -0
  178. package/dist/search/SearchManager.js.map +1 -0
  179. package/dist/search/SearchSuggestions.d.ts +27 -0
  180. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  181. package/dist/search/SearchSuggestions.js +58 -0
  182. package/dist/search/SearchSuggestions.js.map +1 -0
  183. package/dist/search/SemanticSearch.d.ts +149 -0
  184. package/dist/search/SemanticSearch.d.ts.map +1 -0
  185. package/dist/search/SemanticSearch.js +324 -0
  186. package/dist/search/SemanticSearch.js.map +1 -0
  187. package/dist/search/SymbolicSearch.d.ts +61 -0
  188. package/dist/search/SymbolicSearch.d.ts.map +1 -0
  189. package/dist/search/SymbolicSearch.js +164 -0
  190. package/dist/search/SymbolicSearch.js.map +1 -0
  191. package/dist/search/TFIDFEventSync.d.ts +85 -0
  192. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  193. package/dist/search/TFIDFEventSync.js +134 -0
  194. package/dist/search/TFIDFEventSync.js.map +1 -0
  195. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  196. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  197. package/dist/search/TFIDFIndexManager.js +433 -0
  198. package/dist/search/TFIDFIndexManager.js.map +1 -0
  199. package/dist/search/VectorStore.d.ts +235 -0
  200. package/dist/search/VectorStore.d.ts.map +1 -0
  201. package/dist/search/VectorStore.js +312 -0
  202. package/dist/search/VectorStore.js.map +1 -0
  203. package/dist/search/index.d.ts +35 -0
  204. package/dist/search/index.d.ts.map +1 -0
  205. package/dist/search/index.js +53 -0
  206. package/dist/search/index.js.map +1 -0
  207. package/dist/types/index.d.ts +13 -0
  208. package/dist/types/index.d.ts.map +1 -0
  209. package/dist/types/index.js +13 -0
  210. package/dist/types/index.js.map +1 -0
  211. package/dist/types/types.d.ts +1811 -0
  212. package/dist/types/types.d.ts.map +1 -0
  213. package/dist/types/types.js +10 -0
  214. package/dist/types/types.js.map +1 -0
  215. package/dist/utils/BatchProcessor.d.ts +271 -0
  216. package/dist/utils/BatchProcessor.d.ts.map +1 -0
  217. package/dist/utils/BatchProcessor.js +377 -0
  218. package/dist/utils/BatchProcessor.js.map +1 -0
  219. package/dist/utils/MemoryMonitor.d.ts +176 -0
  220. package/dist/utils/MemoryMonitor.d.ts.map +1 -0
  221. package/dist/utils/MemoryMonitor.js +306 -0
  222. package/dist/utils/MemoryMonitor.js.map +1 -0
  223. package/dist/utils/WorkerPoolManager.d.ts +233 -0
  224. package/dist/utils/WorkerPoolManager.d.ts.map +1 -0
  225. package/dist/utils/WorkerPoolManager.js +421 -0
  226. package/dist/utils/WorkerPoolManager.js.map +1 -0
  227. package/dist/utils/compressedCache.d.ts +221 -0
  228. package/dist/utils/compressedCache.d.ts.map +1 -0
  229. package/dist/utils/compressedCache.js +349 -0
  230. package/dist/utils/compressedCache.js.map +1 -0
  231. package/dist/utils/compressionUtil.d.ts +214 -0
  232. package/dist/utils/compressionUtil.d.ts.map +1 -0
  233. package/dist/utils/compressionUtil.js +248 -0
  234. package/dist/utils/compressionUtil.js.map +1 -0
  235. package/dist/utils/constants.d.ts +245 -0
  236. package/dist/utils/constants.d.ts.map +1 -0
  237. package/dist/utils/constants.js +253 -0
  238. package/dist/utils/constants.js.map +1 -0
  239. package/dist/utils/entityUtils.d.ts +379 -0
  240. package/dist/utils/entityUtils.d.ts.map +1 -0
  241. package/dist/utils/entityUtils.js +649 -0
  242. package/dist/utils/entityUtils.js.map +1 -0
  243. package/dist/utils/errors.d.ts +95 -0
  244. package/dist/utils/errors.d.ts.map +1 -0
  245. package/dist/utils/errors.js +146 -0
  246. package/dist/utils/errors.js.map +1 -0
  247. package/dist/utils/formatters.d.ts +145 -0
  248. package/dist/utils/formatters.d.ts.map +1 -0
  249. package/dist/utils/formatters.js +133 -0
  250. package/dist/utils/formatters.js.map +1 -0
  251. package/dist/utils/index.d.ts +26 -0
  252. package/dist/utils/index.d.ts.map +1 -0
  253. package/dist/utils/index.js +88 -0
  254. package/dist/utils/index.js.map +1 -0
  255. package/dist/utils/indexes.d.ts +270 -0
  256. package/dist/utils/indexes.d.ts.map +1 -0
  257. package/dist/utils/indexes.js +527 -0
  258. package/dist/utils/indexes.js.map +1 -0
  259. package/dist/utils/logger.d.ts +31 -0
  260. package/dist/utils/logger.d.ts.map +1 -0
  261. package/dist/utils/logger.js +41 -0
  262. package/dist/utils/logger.js.map +1 -0
  263. package/dist/utils/operationUtils.d.ts +124 -0
  264. package/dist/utils/operationUtils.d.ts.map +1 -0
  265. package/dist/utils/operationUtils.js +176 -0
  266. package/dist/utils/operationUtils.js.map +1 -0
  267. package/dist/utils/parallelUtils.d.ts +76 -0
  268. package/dist/utils/parallelUtils.d.ts.map +1 -0
  269. package/dist/utils/parallelUtils.js +192 -0
  270. package/dist/utils/parallelUtils.js.map +1 -0
  271. package/dist/utils/schemas.d.ts +556 -0
  272. package/dist/utils/schemas.d.ts.map +1 -0
  273. package/dist/utils/schemas.js +485 -0
  274. package/dist/utils/schemas.js.map +1 -0
  275. package/dist/utils/searchAlgorithms.d.ts +99 -0
  276. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  277. package/dist/utils/searchAlgorithms.js +168 -0
  278. package/dist/utils/searchAlgorithms.js.map +1 -0
  279. package/dist/utils/searchCache.d.ts +108 -0
  280. package/dist/utils/searchCache.d.ts.map +1 -0
  281. package/dist/utils/searchCache.js +210 -0
  282. package/dist/utils/searchCache.js.map +1 -0
  283. package/dist/utils/taskScheduler.d.ts +294 -0
  284. package/dist/utils/taskScheduler.d.ts.map +1 -0
  285. package/dist/utils/taskScheduler.js +487 -0
  286. package/dist/utils/taskScheduler.js.map +1 -0
  287. package/dist/workers/index.d.ts +12 -0
  288. package/dist/workers/index.d.ts.map +1 -0
  289. package/dist/workers/index.js +10 -0
  290. package/dist/workers/index.js.map +1 -0
  291. package/dist/workers/levenshteinWorker.d.ts +60 -0
  292. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  293. package/dist/workers/levenshteinWorker.js +99 -0
  294. package/dist/workers/levenshteinWorker.js.map +1 -0
  295. package/package.json +69 -0
@@ -0,0 +1,531 @@
1
+ /**
2
+ * Embedding Service
3
+ *
4
+ * Phase 4 Sprint 10: Provides embedding abstractions for semantic search.
5
+ * Phase 12 Sprint 5: Added query/document prefixes, l2Normalize, mode parameter,
6
+ * and batch embedding with progress callback.
7
+ * Supports multiple providers: OpenAI (cloud) and local (transformers.js).
8
+ *
9
+ * @module search/EmbeddingService
10
+ */
11
+ import { EMBEDDING_DEFAULTS, OPENAI_API_CONFIG, getEmbeddingConfig, } from '../utils/constants.js';
12
+ /**
13
+ * Phase 12 Sprint 5: Prefixes for query-optimized embedding encoding.
14
+ *
15
+ * These prefixes are used to distinguish between query and document embeddings,
16
+ * which can improve retrieval performance with asymmetric embedding models.
17
+ */
18
+ export const QUERY_PREFIX = 'query: ';
19
+ export const DOCUMENT_PREFIX = 'passage: ';
20
+ /**
21
+ * Phase 12 Sprint 5: L2 normalize a vector for cosine similarity.
22
+ *
23
+ * Normalizes a vector to unit length (magnitude 1), which is required
24
+ * for accurate cosine similarity calculations.
25
+ *
26
+ * @param vector - Input vector to normalize
27
+ * @returns L2 normalized vector
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const normalized = l2Normalize([3, 4]); // [0.6, 0.8]
32
+ * ```
33
+ */
34
+ export function l2Normalize(vector) {
35
+ let magnitude = 0;
36
+ for (const v of vector) {
37
+ magnitude += v * v;
38
+ }
39
+ magnitude = Math.sqrt(magnitude);
40
+ if (magnitude === 0 || magnitude === 1) {
41
+ return magnitude === 0 ? vector : vector.slice();
42
+ }
43
+ return vector.map(v => v / magnitude);
44
+ }
45
+ /**
46
+ * OpenAI Embedding Service
47
+ *
48
+ * Uses OpenAI's text-embedding-3-small model for generating embeddings.
49
+ * Supports single and batch embedding with rate limit handling.
50
+ * Phase 12 Sprint 5: Added mode parameter and progress callback support.
51
+ *
52
+ * @example
53
+ * ```typescript
54
+ * const service = new OpenAIEmbeddingService('sk-...');
55
+ * const embedding = await service.embed("Hello world", 'query');
56
+ * console.log(`Generated ${embedding.length} dimensions`);
57
+ * ```
58
+ */
59
+ export class OpenAIEmbeddingService {
60
+ dimensions;
61
+ provider = 'openai';
62
+ model;
63
+ apiKey;
64
+ /**
65
+ * Create an OpenAI embedding service.
66
+ *
67
+ * @param apiKey - OpenAI API key
68
+ * @param model - Optional model override (default: text-embedding-3-small)
69
+ */
70
+ constructor(apiKey, model) {
71
+ if (!apiKey) {
72
+ throw new Error('OpenAI API key is required');
73
+ }
74
+ this.apiKey = apiKey;
75
+ this.model = model || EMBEDDING_DEFAULTS.OPENAI_MODEL;
76
+ this.dimensions = EMBEDDING_DEFAULTS.OPENAI_DIMENSIONS;
77
+ }
78
+ /**
79
+ * Check if the service is ready.
80
+ */
81
+ async isReady() {
82
+ return !!this.apiKey;
83
+ }
84
+ /**
85
+ * Apply prefix to text based on embedding mode.
86
+ *
87
+ * @param text - Original text
88
+ * @param mode - Embedding mode ('query' or 'document')
89
+ * @returns Text with appropriate prefix
90
+ */
91
+ applyPrefix(text, mode = 'document') {
92
+ return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
93
+ }
94
+ /**
95
+ * Generate embedding for a single text.
96
+ *
97
+ * @param text - Text to embed
98
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
99
+ * @returns Embedding vector
100
+ */
101
+ async embed(text, mode = 'document') {
102
+ const results = await this.embedBatch([text], mode);
103
+ return results[0];
104
+ }
105
+ /**
106
+ * Generate embeddings for multiple texts in batch.
107
+ *
108
+ * @param texts - Array of texts to embed
109
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
110
+ * @returns Array of embedding vectors
111
+ */
112
+ async embedBatch(texts, mode = 'document') {
113
+ if (texts.length === 0) {
114
+ return [];
115
+ }
116
+ // Apply prefix based on mode
117
+ const prefixedTexts = texts.map(text => this.applyPrefix(text, mode));
118
+ // Split into batches if needed
119
+ const maxBatchSize = EMBEDDING_DEFAULTS.OPENAI_MAX_BATCH_SIZE;
120
+ const results = [];
121
+ for (let i = 0; i < prefixedTexts.length; i += maxBatchSize) {
122
+ const batch = prefixedTexts.slice(i, i + maxBatchSize);
123
+ const batchResults = await this.embedBatchInternal(batch);
124
+ results.push(...batchResults);
125
+ }
126
+ return results;
127
+ }
128
+ /**
129
+ * Generate embeddings with progress callback support.
130
+ *
131
+ * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
132
+ *
133
+ * @param texts - Array of texts to embed
134
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
135
+ * @param onProgress - Optional progress callback
136
+ * @returns Array of embedding vectors
137
+ */
138
+ async embedBatchWithProgress(texts, mode = 'document', onProgress) {
139
+ if (texts.length === 0) {
140
+ return [];
141
+ }
142
+ // Apply prefix based on mode
143
+ const prefixedTexts = texts.map(text => this.applyPrefix(text, mode));
144
+ // Split into batches if needed
145
+ const maxBatchSize = EMBEDDING_DEFAULTS.OPENAI_MAX_BATCH_SIZE;
146
+ const results = [];
147
+ const total = prefixedTexts.length;
148
+ for (let i = 0; i < prefixedTexts.length; i += maxBatchSize) {
149
+ const batch = prefixedTexts.slice(i, i + maxBatchSize);
150
+ const batchResults = await this.embedBatchInternal(batch);
151
+ results.push(...batchResults);
152
+ // Report progress
153
+ if (onProgress) {
154
+ const current = Math.min(i + maxBatchSize, total);
155
+ onProgress({
156
+ current,
157
+ total,
158
+ percentage: Math.round((current / total) * 100),
159
+ });
160
+ }
161
+ }
162
+ return results;
163
+ }
164
+ /**
165
+ * Internal batch embedding with retry logic.
166
+ */
167
+ async embedBatchInternal(texts) {
168
+ let lastError = null;
169
+ let backoff = OPENAI_API_CONFIG.INITIAL_BACKOFF_MS;
170
+ for (let attempt = 0; attempt <= OPENAI_API_CONFIG.MAX_RETRIES; attempt++) {
171
+ try {
172
+ const response = await fetch(`${OPENAI_API_CONFIG.BASE_URL}${OPENAI_API_CONFIG.EMBEDDINGS_ENDPOINT}`, {
173
+ method: 'POST',
174
+ headers: {
175
+ 'Content-Type': 'application/json',
176
+ 'Authorization': `Bearer ${this.apiKey}`,
177
+ },
178
+ body: JSON.stringify({
179
+ model: this.model,
180
+ input: texts,
181
+ }),
182
+ });
183
+ if (!response.ok) {
184
+ const errorBody = await response.text();
185
+ // Handle rate limiting
186
+ if (response.status === 429) {
187
+ if (attempt < OPENAI_API_CONFIG.MAX_RETRIES) {
188
+ await this.sleep(backoff);
189
+ backoff = Math.min(backoff * 2, OPENAI_API_CONFIG.MAX_BACKOFF_MS);
190
+ continue;
191
+ }
192
+ }
193
+ throw new Error(`OpenAI API error: ${response.status} - ${errorBody}`);
194
+ }
195
+ const data = await response.json();
196
+ // Sort by index to ensure correct order
197
+ const sortedData = [...data.data].sort((a, b) => a.index - b.index);
198
+ return sortedData.map(item => item.embedding);
199
+ }
200
+ catch (error) {
201
+ lastError = error instanceof Error ? error : new Error(String(error));
202
+ // Retry on network errors
203
+ if (attempt < OPENAI_API_CONFIG.MAX_RETRIES && this.isRetryableError(error)) {
204
+ await this.sleep(backoff);
205
+ backoff = Math.min(backoff * 2, OPENAI_API_CONFIG.MAX_BACKOFF_MS);
206
+ continue;
207
+ }
208
+ throw lastError;
209
+ }
210
+ }
211
+ throw lastError || new Error('Failed to generate embeddings after retries');
212
+ }
213
+ /**
214
+ * Check if an error is retryable.
215
+ */
216
+ isRetryableError(error) {
217
+ if (error instanceof Error) {
218
+ // Network errors and rate limits are retryable
219
+ return error.message.includes('fetch') ||
220
+ error.message.includes('network') ||
221
+ error.message.includes('429');
222
+ }
223
+ return false;
224
+ }
225
+ /**
226
+ * Sleep for a given duration.
227
+ */
228
+ sleep(ms) {
229
+ return new Promise(resolve => setTimeout(resolve, ms));
230
+ }
231
+ }
232
+ /**
233
+ * Local Embedding Service
234
+ *
235
+ * Uses @xenova/transformers for local embedding generation.
236
+ * No API calls needed - runs entirely offline after initial model download.
237
+ * Phase 12 Sprint 5: Added mode parameter and progress callback support.
238
+ *
239
+ * Note: Requires @xenova/transformers to be installed as an optional dependency.
240
+ * If not available, initialization will fail gracefully.
241
+ *
242
+ * @example
243
+ * ```typescript
244
+ * const service = new LocalEmbeddingService();
245
+ * await service.initialize();
246
+ * const embedding = await service.embed("Hello world", 'query');
247
+ * ```
248
+ */
249
+ export class LocalEmbeddingService {
250
+ dimensions = EMBEDDING_DEFAULTS.LOCAL_DIMENSIONS;
251
+ provider = 'local';
252
+ model;
253
+ pipeline = null;
254
+ initialized = false;
255
+ initPromise = null;
256
+ /**
257
+ * Create a local embedding service.
258
+ *
259
+ * @param model - Optional model override (default: Xenova/all-MiniLM-L6-v2)
260
+ */
261
+ constructor(model) {
262
+ this.model = model || EMBEDDING_DEFAULTS.LOCAL_MODEL;
263
+ }
264
+ /**
265
+ * Initialize the model pipeline.
266
+ * Must be called before using embed/embedBatch.
267
+ */
268
+ async initialize() {
269
+ if (this.initialized)
270
+ return;
271
+ if (this.initPromise) {
272
+ return this.initPromise;
273
+ }
274
+ this.initPromise = this.initializeInternal();
275
+ return this.initPromise;
276
+ }
277
+ /**
278
+ * Internal initialization.
279
+ */
280
+ async initializeInternal() {
281
+ try {
282
+ // Dynamic import to allow optional dependency
283
+ // @ts-expect-error - @xenova/transformers is an optional peer dependency
284
+ const transformers = await import('@xenova/transformers');
285
+ const { pipeline } = transformers;
286
+ this.pipeline = await pipeline('feature-extraction', this.model);
287
+ this.initialized = true;
288
+ }
289
+ catch (error) {
290
+ this.initPromise = null;
291
+ throw new Error(`Failed to initialize local embedding service: ${error instanceof Error ? error.message : String(error)}. ` +
292
+ 'Make sure @xenova/transformers is installed.');
293
+ }
294
+ }
295
+ /**
296
+ * Check if the service is ready.
297
+ */
298
+ async isReady() {
299
+ if (!this.initialized && !this.initPromise) {
300
+ try {
301
+ await this.initialize();
302
+ }
303
+ catch {
304
+ return false;
305
+ }
306
+ }
307
+ return this.initialized;
308
+ }
309
+ /**
310
+ * Apply prefix to text based on embedding mode.
311
+ *
312
+ * @param text - Original text
313
+ * @param mode - Embedding mode ('query' or 'document')
314
+ * @returns Text with appropriate prefix
315
+ */
316
+ applyPrefix(text, mode = 'document') {
317
+ return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
318
+ }
319
+ /**
320
+ * Generate embedding for a single text.
321
+ *
322
+ * @param text - Text to embed
323
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
324
+ * @returns Embedding vector
325
+ */
326
+ async embed(text, mode = 'document') {
327
+ await this.ensureInitialized();
328
+ const prefixedText = this.applyPrefix(text, mode);
329
+ const pipelineFn = this.pipeline;
330
+ const output = await pipelineFn(prefixedText, { pooling: 'mean', normalize: true });
331
+ return Array.from(output.data);
332
+ }
333
+ /**
334
+ * Generate embeddings for multiple texts in batch.
335
+ * Note: Local processing is done sequentially to avoid memory issues.
336
+ *
337
+ * @param texts - Array of texts to embed
338
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
339
+ * @returns Array of embedding vectors
340
+ */
341
+ async embedBatch(texts, mode = 'document') {
342
+ await this.ensureInitialized();
343
+ const results = [];
344
+ for (const text of texts) {
345
+ const embedding = await this.embed(text, mode);
346
+ results.push(embedding);
347
+ }
348
+ return results;
349
+ }
350
+ /**
351
+ * Generate embeddings with progress callback support.
352
+ *
353
+ * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
354
+ *
355
+ * @param texts - Array of texts to embed
356
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
357
+ * @param onProgress - Optional progress callback
358
+ * @returns Array of embedding vectors
359
+ */
360
+ async embedBatchWithProgress(texts, mode = 'document', onProgress) {
361
+ await this.ensureInitialized();
362
+ const results = [];
363
+ const total = texts.length;
364
+ for (let i = 0; i < texts.length; i++) {
365
+ const embedding = await this.embed(texts[i], mode);
366
+ results.push(embedding);
367
+ // Report progress
368
+ if (onProgress) {
369
+ const current = i + 1;
370
+ onProgress({
371
+ current,
372
+ total,
373
+ percentage: Math.round((current / total) * 100),
374
+ });
375
+ }
376
+ }
377
+ return results;
378
+ }
379
+ /**
380
+ * Ensure the service is initialized.
381
+ */
382
+ async ensureInitialized() {
383
+ if (!this.initialized) {
384
+ await this.initialize();
385
+ }
386
+ }
387
+ }
388
+ /**
389
+ * Mock Embedding Service for testing
390
+ *
391
+ * Generates deterministic mock embeddings for testing purposes.
392
+ * Useful for unit tests that don't need real embeddings.
393
+ * Phase 12 Sprint 5: Added mode parameter and progress callback support.
394
+ */
395
+ export class MockEmbeddingService {
396
+ dimensions;
397
+ provider = 'mock';
398
+ model = 'mock-model';
399
+ /**
400
+ * Create a mock embedding service.
401
+ *
402
+ * @param dimensions - Number of dimensions for mock embeddings
403
+ */
404
+ constructor(dimensions = 384) {
405
+ this.dimensions = dimensions;
406
+ }
407
+ /**
408
+ * Check if the service is ready.
409
+ */
410
+ async isReady() {
411
+ return true;
412
+ }
413
+ /**
414
+ * Apply prefix to text based on embedding mode.
415
+ *
416
+ * @param text - Original text
417
+ * @param mode - Embedding mode ('query' or 'document')
418
+ * @returns Text with appropriate prefix
419
+ */
420
+ applyPrefix(text, mode = 'document') {
421
+ return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
422
+ }
423
+ /**
424
+ * Generate a deterministic mock embedding for a text.
425
+ *
426
+ * @param text - Text to embed
427
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
428
+ * @returns Mock embedding vector
429
+ */
430
+ async embed(text, mode = 'document') {
431
+ // Apply prefix based on mode (affects hash for different embeddings per mode)
432
+ const prefixedText = this.applyPrefix(text, mode);
433
+ // Generate deterministic embedding based on text hash
434
+ const hash = this.hashString(prefixedText);
435
+ const embedding = [];
436
+ for (let i = 0; i < this.dimensions; i++) {
437
+ // Use hash and index to generate deterministic values
438
+ const value = Math.sin(hash + i * 0.1) * 0.5;
439
+ embedding.push(value);
440
+ }
441
+ // Normalize the vector
442
+ return this.normalize(embedding);
443
+ }
444
+ /**
445
+ * Generate mock embeddings for multiple texts.
446
+ *
447
+ * @param texts - Array of texts to embed
448
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
449
+ * @returns Array of mock embedding vectors
450
+ */
451
+ async embedBatch(texts, mode = 'document') {
452
+ return Promise.all(texts.map(text => this.embed(text, mode)));
453
+ }
454
+ /**
455
+ * Generate mock embeddings with progress callback support.
456
+ *
457
+ * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
458
+ *
459
+ * @param texts - Array of texts to embed
460
+ * @param mode - Embedding mode ('query' or 'document', default: 'document')
461
+ * @param onProgress - Optional progress callback
462
+ * @returns Array of mock embedding vectors
463
+ */
464
+ async embedBatchWithProgress(texts, mode = 'document', onProgress) {
465
+ const results = [];
466
+ const total = texts.length;
467
+ for (let i = 0; i < texts.length; i++) {
468
+ const embedding = await this.embed(texts[i], mode);
469
+ results.push(embedding);
470
+ // Report progress
471
+ if (onProgress) {
472
+ const current = i + 1;
473
+ onProgress({
474
+ current,
475
+ total,
476
+ percentage: Math.round((current / total) * 100),
477
+ });
478
+ }
479
+ }
480
+ return results;
481
+ }
482
+ /**
483
+ * Simple string hash function.
484
+ */
485
+ hashString(str) {
486
+ let hash = 0;
487
+ for (let i = 0; i < str.length; i++) {
488
+ const char = str.charCodeAt(i);
489
+ hash = ((hash << 5) - hash) + char;
490
+ hash = hash & hash; // Convert to 32bit integer
491
+ }
492
+ return hash;
493
+ }
494
+ /**
495
+ * Normalize a vector to unit length.
496
+ */
497
+ normalize(vector) {
498
+ let magnitude = 0;
499
+ for (const v of vector) {
500
+ magnitude += v * v;
501
+ }
502
+ magnitude = Math.sqrt(magnitude);
503
+ if (magnitude === 0) {
504
+ return vector;
505
+ }
506
+ return vector.map(v => v / magnitude);
507
+ }
508
+ }
509
+ /**
510
+ * Create an embedding service based on configuration.
511
+ *
512
+ * @param config - Optional configuration override
513
+ * @returns Embedding service instance, or null if provider is 'none'
514
+ */
515
+ export function createEmbeddingService(config) {
516
+ const envConfig = getEmbeddingConfig();
517
+ const mergedConfig = { ...envConfig, ...config };
518
+ switch (mergedConfig.provider) {
519
+ case 'openai':
520
+ if (!mergedConfig.apiKey) {
521
+ throw new Error('OpenAI API key is required. Set MEMORY_OPENAI_API_KEY environment variable or provide apiKey in config.');
522
+ }
523
+ return new OpenAIEmbeddingService(mergedConfig.apiKey, mergedConfig.model);
524
+ case 'local':
525
+ return new LocalEmbeddingService(mergedConfig.model);
526
+ case 'none':
527
+ default:
528
+ return null;
529
+ }
530
+ }
531
+ //# sourceMappingURL=EmbeddingService.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"EmbeddingService.js","sourceRoot":"","sources":["../../src/search/EmbeddingService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,SAAS,CAAC;AACtC,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC;AAW3C;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,MAAgB;IAC1C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEjC,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,sBAAsB;IACxB,UAAU,CAAS;IACnB,QAAQ,GAAG,QAAQ,CAAC;IACpB,KAAK,CAAS;IACf,MAAM,CAAS;IAEvB;;;;;OAKG;IACH,YAAY,MAAc,EAAE,KAAc;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,kBAAkB,CAAC,YAAY,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,6BAA6B;QAC7B,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEtE,+BAA+B;QAC/B,MAAM,YAAY,GAAG,kBAAkB,CAAC,qBAAqB,CAAC;QAC9D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC;YACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,6BAA6B;QAC7B,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEtE,+BAA+B;QAC/B,MAAM,YAAY,GAAG,kBAAkB,CAAC,qBAAqB,CAAC;QAC9D,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC;YACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAE9B,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,CAAC;gBAClD,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAAC,KAAe;QAC9C,IAAI,SAAS,GAAiB,IAAI,CAAC;QACnC,IAAI,OAAO,GAAG,iBAAiB,CAAC,kBAAkB,CAAC;QAEnD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,GAAG,iBAAiB,CAAC,QAAQ,GAAG,iBAAiB,CAAC,mBAAmB,EAAE,EACvE;oBACE,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACzC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,KAAK,EAAE,KAAK;qBACb,CAAC;iBACH,CACF,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAExC,uBAAuB;oBACvB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC5B,IAAI,OAAO,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;4BAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;4BAC1B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;4BAClE,SAAS;wBACX,CAAC;oBACH,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;gBACzE,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;gBAE9D,wCAAwC;gBACxC,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBACpE,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,0BAA0B;gBAC1B,IAAI,OAAO,GAAG,iBAAiB,CAAC,WAAW,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC5E,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC1B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,MAAM,SAAS,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,+CAA+C;YAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAC/B,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACjC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF;AAmBD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,qBAAqB;IACvB,UAAU,GAAW,kBAAkB,CAAC,gBAAgB,CAAC;IACzD,QAAQ,GAAG,OAAO,CAAC;IACnB,KAAK,CAAS;IAEf,QAAQ,GAAY,IAAI,CAAC;IACzB,WAAW,GAAG,KAAK,CAAC;IACpB,WAAW,GAAyB,IAAI,CAAC;IAEjD;;;;OAIG;IACH,YAAY,KAAc;QACxB,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,kBAAkB,CAAC,WAAW,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC;YACH,8CAA8C;YAC9C,yEAAyE;YACzE,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC1D,MAAM,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC;YAElC,IAAI,CAAC,QAAQ,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,iDAAiD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBAC3G,8CAA8C,CAC/C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,QAA+G,CAAC;QACxI,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExB,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtB,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,oBAAoB;IACtB,UAAU,CAAS;IACnB,QAAQ,GAAG,MAAM,CAAC;IAClB,KAAK,GAAG,YAAY,CAAC;IAE9B;;;;OAIG;IACH,YAAY,aAAqB,GAAG;QAClC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,8EAA8E;QAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAElD,sDAAsD;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,sDAAsD;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QAED,uBAAuB;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExB,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtB,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;QACjD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,MAAgB;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IACxC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAiC;IACtE,MAAM,SAAS,GAAG,kBAAkB,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,EAAE,GAAG,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;IAEjD,QAAQ,YAAY,CAAC,QAAQ,EAAE,CAAC;QAC9B,KAAK,QAAQ;YACX,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CACb,yGAAyG,CAC1G,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,sBAAsB,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAE7E,KAAK,OAAO;YACV,OAAO,IAAI,qBAAqB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAEvD,KAAK,MAAM,CAAC;QACZ;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
@@ -0,0 +1,118 @@
1
+ /**
2
+ * Fuzzy Search
3
+ *
4
+ * Search with typo tolerance using Levenshtein distance similarity.
5
+ * Uses workerpool for parallel processing on large datasets.
6
+ *
7
+ * @module search/FuzzySearch
8
+ */
9
+ import type { KnowledgeGraph } from '../types/index.js';
10
+ import type { GraphStorage } from '../core/GraphStorage.js';
11
+ /**
12
+ * Default fuzzy search similarity threshold (70% match required).
13
+ * Lower values are more permissive (more typos tolerated).
14
+ * Higher values are stricter (fewer typos tolerated).
15
+ */
16
+ export declare const DEFAULT_FUZZY_THRESHOLD = 0.7;
17
+ /**
18
+ * Options for FuzzySearch constructor.
19
+ */
20
+ export interface FuzzySearchOptions {
21
+ /**
22
+ * Whether to use worker pool for parallel processing.
23
+ * Set to false for testing or when workers are not available.
24
+ * Default: true
25
+ */
26
+ useWorkerPool?: boolean;
27
+ }
28
+ /**
29
+ * Performs fuzzy search with configurable similarity threshold.
30
+ */
31
+ export declare class FuzzySearch {
32
+ private storage;
33
+ /**
34
+ * Phase 4 Sprint 3: Result cache for fuzzy search.
35
+ * Maps cache key -> cached entity names.
36
+ */
37
+ private fuzzyResultCache;
38
+ /**
39
+ * Phase 8: Worker pool using workerpool library.
40
+ * Initialized lazily when needed.
41
+ */
42
+ private workerPool;
43
+ /**
44
+ * Phase 7 Sprint 3: Path to the worker script.
45
+ */
46
+ private workerPath;
47
+ /**
48
+ * Phase 8: Whether to use worker pool for parallel processing.
49
+ * Can be disabled for testing or when workers are not available.
50
+ */
51
+ private useWorkerPool;
52
+ constructor(storage: GraphStorage, options?: FuzzySearchOptions);
53
+ /**
54
+ * Phase 4 Sprint 3: Generate cache key for fuzzy search parameters.
55
+ */
56
+ private generateCacheKey;
57
+ /**
58
+ * Phase 4 Sprint 3: Clear the fuzzy search cache.
59
+ */
60
+ clearCache(): void;
61
+ /**
62
+ * Phase 4 Sprint 3: Invalidate stale cache entries.
63
+ */
64
+ private cleanupCache;
65
+ /**
66
+ * Fuzzy search for entities with typo tolerance and pagination.
67
+ *
68
+ * Uses Levenshtein distance to calculate similarity between strings.
69
+ * Matches if similarity >= threshold (0.0 to 1.0).
70
+ *
71
+ * Phase 4 Sprint 3: Implements result caching for repeated queries.
72
+ *
73
+ * @param query - Search query
74
+ * @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_FUZZY_THRESHOLD
75
+ * @param tags - Optional tags filter
76
+ * @param minImportance - Optional minimum importance
77
+ * @param maxImportance - Optional maximum importance
78
+ * @param offset - Number of results to skip (default: 0)
79
+ * @param limit - Maximum number of results (default: 50, max: 200)
80
+ * @returns Filtered knowledge graph with fuzzy matches and pagination applied
81
+ */
82
+ fuzzySearch(query: string, threshold?: number, tags?: string[], minImportance?: number, maxImportance?: number, offset?: number, limit?: number): Promise<KnowledgeGraph>;
83
+ /**
84
+ * Phase 4 Sprint 3: Perform the actual fuzzy matching logic.
85
+ * Extracted from fuzzySearch for cleaner code structure.
86
+ */
87
+ private performFuzzyMatch;
88
+ /**
89
+ * Check if two already-lowercase strings match with fuzzy logic.
90
+ *
91
+ * OPTIMIZED: Skips toLowerCase() calls when strings are already lowercase.
92
+ *
93
+ * @param s1 - First string (already lowercase)
94
+ * @param s2 - Second string (already lowercase)
95
+ * @param threshold - Similarity threshold (0.0 to 1.0)
96
+ * @returns True if strings match fuzzily
97
+ */
98
+ private isFuzzyMatchLower;
99
+ /**
100
+ * Phase 8: Perform fuzzy search using workerpool for parallel processing.
101
+ *
102
+ * Splits entities into chunks and processes them in parallel using worker threads.
103
+ * Falls back to single-threaded search if worker execution fails.
104
+ *
105
+ * @param query - Search query
106
+ * @param threshold - Similarity threshold
107
+ * @param entities - Entities to search
108
+ * @returns Array of matched entities
109
+ */
110
+ private searchWithWorkers;
111
+ /**
112
+ * Phase 8: Shutdown the worker pool and clean up resources.
113
+ *
114
+ * Should be called when FuzzySearch is no longer needed.
115
+ */
116
+ shutdown(): Promise<void>;
117
+ }
118
+ //# sourceMappingURL=FuzzySearch.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FuzzySearch.d.ts","sourceRoot":"","sources":["../../src/search/FuzzySearch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAU,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAQ5D;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AA4C3C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,qBAAa,WAAW;IAwBV,OAAO,CAAC,OAAO;IAvB3B;;;OAGG;IACH,OAAO,CAAC,gBAAgB,CAA2C;IAEnE;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAqB;IAEvC;;OAEG;IACH,OAAO,CAAC,UAAU,CAAS;IAE3B;;;OAGG;IACH,OAAO,CAAC,aAAa,CAAU;gBAEX,OAAO,EAAE,YAAY,EAAE,OAAO,GAAE,kBAAuB;IAmB3E;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAoBxB;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAwBpB;;;;;;;;;;;;;;;;OAgBG;IACG,WAAW,CACf,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAgC,EAC3C,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,aAAa,CAAC,EAAE,MAAM,EACtB,aAAa,CAAC,EAAE,MAAM,EACtB,MAAM,GAAE,MAAU,EAClB,KAAK,GAAE,MAA8B,GACpC,OAAO,CAAC,cAAc,CAAC;IAsE1B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA0BzB;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;IAezB;;;;;;;;;;OAUG;YACW,iBAAiB;IAiE/B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAMhC"}