@danielsimonjr/memoryjs 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/README.md +385 -113
  2. package/README.md.backup-1768084780988 +266 -0
  3. package/dist/index.cjs +24156 -0
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.cts +16967 -0
  6. package/dist/index.d.ts +16963 -11
  7. package/dist/index.js +23887 -19
  8. package/dist/index.js.map +1 -1
  9. package/dist/workers/levenshteinWorker.cjs +102 -0
  10. package/dist/workers/levenshteinWorker.cjs.map +1 -0
  11. package/dist/workers/levenshteinWorker.js +57 -91
  12. package/dist/workers/levenshteinWorker.js.map +1 -1
  13. package/package.json +75 -69
  14. package/dist/core/EntityManager.d.ts +0 -268
  15. package/dist/core/EntityManager.d.ts.map +0 -1
  16. package/dist/core/EntityManager.js +0 -512
  17. package/dist/core/EntityManager.js.map +0 -1
  18. package/dist/core/GraphEventEmitter.d.ts +0 -202
  19. package/dist/core/GraphEventEmitter.d.ts.map +0 -1
  20. package/dist/core/GraphEventEmitter.js +0 -347
  21. package/dist/core/GraphEventEmitter.js.map +0 -1
  22. package/dist/core/GraphStorage.d.ts +0 -395
  23. package/dist/core/GraphStorage.d.ts.map +0 -1
  24. package/dist/core/GraphStorage.js +0 -786
  25. package/dist/core/GraphStorage.js.map +0 -1
  26. package/dist/core/GraphTraversal.d.ts +0 -141
  27. package/dist/core/GraphTraversal.d.ts.map +0 -1
  28. package/dist/core/GraphTraversal.js +0 -574
  29. package/dist/core/GraphTraversal.js.map +0 -1
  30. package/dist/core/HierarchyManager.d.ts +0 -111
  31. package/dist/core/HierarchyManager.d.ts.map +0 -1
  32. package/dist/core/HierarchyManager.js +0 -225
  33. package/dist/core/HierarchyManager.js.map +0 -1
  34. package/dist/core/ManagerContext.d.ts +0 -76
  35. package/dist/core/ManagerContext.d.ts.map +0 -1
  36. package/dist/core/ManagerContext.js +0 -129
  37. package/dist/core/ManagerContext.js.map +0 -1
  38. package/dist/core/ObservationManager.d.ts +0 -85
  39. package/dist/core/ObservationManager.d.ts.map +0 -1
  40. package/dist/core/ObservationManager.js +0 -124
  41. package/dist/core/ObservationManager.js.map +0 -1
  42. package/dist/core/RelationManager.d.ts +0 -131
  43. package/dist/core/RelationManager.d.ts.map +0 -1
  44. package/dist/core/RelationManager.js +0 -212
  45. package/dist/core/RelationManager.js.map +0 -1
  46. package/dist/core/SQLiteStorage.d.ts +0 -354
  47. package/dist/core/SQLiteStorage.d.ts.map +0 -1
  48. package/dist/core/SQLiteStorage.js +0 -919
  49. package/dist/core/SQLiteStorage.js.map +0 -1
  50. package/dist/core/StorageFactory.d.ts +0 -45
  51. package/dist/core/StorageFactory.d.ts.map +0 -1
  52. package/dist/core/StorageFactory.js +0 -65
  53. package/dist/core/StorageFactory.js.map +0 -1
  54. package/dist/core/TransactionManager.d.ts +0 -464
  55. package/dist/core/TransactionManager.d.ts.map +0 -1
  56. package/dist/core/TransactionManager.js +0 -869
  57. package/dist/core/TransactionManager.js.map +0 -1
  58. package/dist/core/index.d.ts +0 -17
  59. package/dist/core/index.d.ts.map +0 -1
  60. package/dist/core/index.js +0 -20
  61. package/dist/core/index.js.map +0 -1
  62. package/dist/features/AnalyticsManager.d.ts +0 -44
  63. package/dist/features/AnalyticsManager.d.ts.map +0 -1
  64. package/dist/features/AnalyticsManager.js +0 -224
  65. package/dist/features/AnalyticsManager.js.map +0 -1
  66. package/dist/features/ArchiveManager.d.ts +0 -133
  67. package/dist/features/ArchiveManager.d.ts.map +0 -1
  68. package/dist/features/ArchiveManager.js +0 -282
  69. package/dist/features/ArchiveManager.js.map +0 -1
  70. package/dist/features/CompressionManager.d.ts +0 -119
  71. package/dist/features/CompressionManager.d.ts.map +0 -1
  72. package/dist/features/CompressionManager.js +0 -470
  73. package/dist/features/CompressionManager.js.map +0 -1
  74. package/dist/features/IOManager.d.ts +0 -225
  75. package/dist/features/IOManager.d.ts.map +0 -1
  76. package/dist/features/IOManager.js +0 -1093
  77. package/dist/features/IOManager.js.map +0 -1
  78. package/dist/features/KeywordExtractor.d.ts +0 -61
  79. package/dist/features/KeywordExtractor.d.ts.map +0 -1
  80. package/dist/features/KeywordExtractor.js +0 -127
  81. package/dist/features/KeywordExtractor.js.map +0 -1
  82. package/dist/features/ObservationNormalizer.d.ts +0 -90
  83. package/dist/features/ObservationNormalizer.d.ts.map +0 -1
  84. package/dist/features/ObservationNormalizer.js +0 -194
  85. package/dist/features/ObservationNormalizer.js.map +0 -1
  86. package/dist/features/StreamingExporter.d.ts +0 -128
  87. package/dist/features/StreamingExporter.d.ts.map +0 -1
  88. package/dist/features/StreamingExporter.js +0 -212
  89. package/dist/features/StreamingExporter.js.map +0 -1
  90. package/dist/features/TagManager.d.ts +0 -147
  91. package/dist/features/TagManager.d.ts.map +0 -1
  92. package/dist/features/TagManager.js +0 -211
  93. package/dist/features/TagManager.js.map +0 -1
  94. package/dist/features/index.d.ts +0 -14
  95. package/dist/features/index.d.ts.map +0 -1
  96. package/dist/features/index.js +0 -15
  97. package/dist/features/index.js.map +0 -1
  98. package/dist/index.d.ts.map +0 -1
  99. package/dist/search/BM25Search.d.ts +0 -148
  100. package/dist/search/BM25Search.d.ts.map +0 -1
  101. package/dist/search/BM25Search.js +0 -340
  102. package/dist/search/BM25Search.js.map +0 -1
  103. package/dist/search/BasicSearch.d.ts +0 -51
  104. package/dist/search/BasicSearch.d.ts.map +0 -1
  105. package/dist/search/BasicSearch.js +0 -138
  106. package/dist/search/BasicSearch.js.map +0 -1
  107. package/dist/search/BooleanSearch.d.ts +0 -98
  108. package/dist/search/BooleanSearch.d.ts.map +0 -1
  109. package/dist/search/BooleanSearch.js +0 -431
  110. package/dist/search/BooleanSearch.js.map +0 -1
  111. package/dist/search/EarlyTerminationManager.d.ts +0 -140
  112. package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
  113. package/dist/search/EarlyTerminationManager.js +0 -280
  114. package/dist/search/EarlyTerminationManager.js.map +0 -1
  115. package/dist/search/EmbeddingCache.d.ts +0 -175
  116. package/dist/search/EmbeddingCache.d.ts.map +0 -1
  117. package/dist/search/EmbeddingCache.js +0 -247
  118. package/dist/search/EmbeddingCache.js.map +0 -1
  119. package/dist/search/EmbeddingService.d.ts +0 -277
  120. package/dist/search/EmbeddingService.d.ts.map +0 -1
  121. package/dist/search/EmbeddingService.js +0 -531
  122. package/dist/search/EmbeddingService.js.map +0 -1
  123. package/dist/search/FuzzySearch.d.ts +0 -118
  124. package/dist/search/FuzzySearch.d.ts.map +0 -1
  125. package/dist/search/FuzzySearch.js +0 -313
  126. package/dist/search/FuzzySearch.js.map +0 -1
  127. package/dist/search/HybridScorer.d.ts +0 -181
  128. package/dist/search/HybridScorer.d.ts.map +0 -1
  129. package/dist/search/HybridScorer.js +0 -258
  130. package/dist/search/HybridScorer.js.map +0 -1
  131. package/dist/search/HybridSearchManager.d.ts +0 -80
  132. package/dist/search/HybridSearchManager.d.ts.map +0 -1
  133. package/dist/search/HybridSearchManager.js +0 -188
  134. package/dist/search/HybridSearchManager.js.map +0 -1
  135. package/dist/search/IncrementalIndexer.d.ts +0 -201
  136. package/dist/search/IncrementalIndexer.d.ts.map +0 -1
  137. package/dist/search/IncrementalIndexer.js +0 -343
  138. package/dist/search/IncrementalIndexer.js.map +0 -1
  139. package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
  140. package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
  141. package/dist/search/OptimizedInvertedIndex.js +0 -359
  142. package/dist/search/OptimizedInvertedIndex.js.map +0 -1
  143. package/dist/search/ParallelSearchExecutor.d.ts +0 -172
  144. package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
  145. package/dist/search/ParallelSearchExecutor.js +0 -310
  146. package/dist/search/ParallelSearchExecutor.js.map +0 -1
  147. package/dist/search/QuantizedVectorStore.d.ts +0 -171
  148. package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
  149. package/dist/search/QuantizedVectorStore.js +0 -308
  150. package/dist/search/QuantizedVectorStore.js.map +0 -1
  151. package/dist/search/QueryAnalyzer.d.ts +0 -76
  152. package/dist/search/QueryAnalyzer.d.ts.map +0 -1
  153. package/dist/search/QueryAnalyzer.js +0 -228
  154. package/dist/search/QueryAnalyzer.js.map +0 -1
  155. package/dist/search/QueryCostEstimator.d.ts +0 -244
  156. package/dist/search/QueryCostEstimator.d.ts.map +0 -1
  157. package/dist/search/QueryCostEstimator.js +0 -653
  158. package/dist/search/QueryCostEstimator.js.map +0 -1
  159. package/dist/search/QueryPlanCache.d.ts +0 -220
  160. package/dist/search/QueryPlanCache.d.ts.map +0 -1
  161. package/dist/search/QueryPlanCache.js +0 -380
  162. package/dist/search/QueryPlanCache.js.map +0 -1
  163. package/dist/search/QueryPlanner.d.ts +0 -58
  164. package/dist/search/QueryPlanner.d.ts.map +0 -1
  165. package/dist/search/QueryPlanner.js +0 -138
  166. package/dist/search/QueryPlanner.js.map +0 -1
  167. package/dist/search/RankedSearch.d.ts +0 -71
  168. package/dist/search/RankedSearch.d.ts.map +0 -1
  169. package/dist/search/RankedSearch.js +0 -239
  170. package/dist/search/RankedSearch.js.map +0 -1
  171. package/dist/search/ReflectionManager.d.ts +0 -120
  172. package/dist/search/ReflectionManager.d.ts.map +0 -1
  173. package/dist/search/ReflectionManager.js +0 -232
  174. package/dist/search/ReflectionManager.js.map +0 -1
  175. package/dist/search/SavedSearchManager.d.ts +0 -79
  176. package/dist/search/SavedSearchManager.d.ts.map +0 -1
  177. package/dist/search/SavedSearchManager.js +0 -147
  178. package/dist/search/SavedSearchManager.js.map +0 -1
  179. package/dist/search/SearchFilterChain.d.ts +0 -120
  180. package/dist/search/SearchFilterChain.d.ts.map +0 -1
  181. package/dist/search/SearchFilterChain.js +0 -186
  182. package/dist/search/SearchFilterChain.js.map +0 -1
  183. package/dist/search/SearchManager.d.ts +0 -326
  184. package/dist/search/SearchManager.d.ts.map +0 -1
  185. package/dist/search/SearchManager.js +0 -454
  186. package/dist/search/SearchManager.js.map +0 -1
  187. package/dist/search/SearchSuggestions.d.ts +0 -27
  188. package/dist/search/SearchSuggestions.d.ts.map +0 -1
  189. package/dist/search/SearchSuggestions.js +0 -58
  190. package/dist/search/SearchSuggestions.js.map +0 -1
  191. package/dist/search/SemanticSearch.d.ts +0 -149
  192. package/dist/search/SemanticSearch.d.ts.map +0 -1
  193. package/dist/search/SemanticSearch.js +0 -324
  194. package/dist/search/SemanticSearch.js.map +0 -1
  195. package/dist/search/SymbolicSearch.d.ts +0 -61
  196. package/dist/search/SymbolicSearch.d.ts.map +0 -1
  197. package/dist/search/SymbolicSearch.js +0 -164
  198. package/dist/search/SymbolicSearch.js.map +0 -1
  199. package/dist/search/TFIDFEventSync.d.ts +0 -85
  200. package/dist/search/TFIDFEventSync.d.ts.map +0 -1
  201. package/dist/search/TFIDFEventSync.js +0 -134
  202. package/dist/search/TFIDFEventSync.js.map +0 -1
  203. package/dist/search/TFIDFIndexManager.d.ts +0 -151
  204. package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
  205. package/dist/search/TFIDFIndexManager.js +0 -433
  206. package/dist/search/TFIDFIndexManager.js.map +0 -1
  207. package/dist/search/VectorStore.d.ts +0 -235
  208. package/dist/search/VectorStore.d.ts.map +0 -1
  209. package/dist/search/VectorStore.js +0 -312
  210. package/dist/search/VectorStore.js.map +0 -1
  211. package/dist/search/index.d.ts +0 -35
  212. package/dist/search/index.d.ts.map +0 -1
  213. package/dist/search/index.js +0 -53
  214. package/dist/search/index.js.map +0 -1
  215. package/dist/types/index.d.ts +0 -13
  216. package/dist/types/index.d.ts.map +0 -1
  217. package/dist/types/index.js +0 -13
  218. package/dist/types/index.js.map +0 -1
  219. package/dist/types/types.d.ts +0 -1811
  220. package/dist/types/types.d.ts.map +0 -1
  221. package/dist/types/types.js +0 -10
  222. package/dist/types/types.js.map +0 -1
  223. package/dist/utils/BatchProcessor.d.ts +0 -271
  224. package/dist/utils/BatchProcessor.d.ts.map +0 -1
  225. package/dist/utils/BatchProcessor.js +0 -377
  226. package/dist/utils/BatchProcessor.js.map +0 -1
  227. package/dist/utils/MemoryMonitor.d.ts +0 -176
  228. package/dist/utils/MemoryMonitor.d.ts.map +0 -1
  229. package/dist/utils/MemoryMonitor.js +0 -306
  230. package/dist/utils/MemoryMonitor.js.map +0 -1
  231. package/dist/utils/WorkerPoolManager.d.ts +0 -233
  232. package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
  233. package/dist/utils/WorkerPoolManager.js +0 -421
  234. package/dist/utils/WorkerPoolManager.js.map +0 -1
  235. package/dist/utils/compressedCache.d.ts +0 -221
  236. package/dist/utils/compressedCache.d.ts.map +0 -1
  237. package/dist/utils/compressedCache.js +0 -349
  238. package/dist/utils/compressedCache.js.map +0 -1
  239. package/dist/utils/compressionUtil.d.ts +0 -214
  240. package/dist/utils/compressionUtil.d.ts.map +0 -1
  241. package/dist/utils/compressionUtil.js +0 -248
  242. package/dist/utils/compressionUtil.js.map +0 -1
  243. package/dist/utils/constants.d.ts +0 -245
  244. package/dist/utils/constants.d.ts.map +0 -1
  245. package/dist/utils/constants.js +0 -253
  246. package/dist/utils/constants.js.map +0 -1
  247. package/dist/utils/entityUtils.d.ts +0 -379
  248. package/dist/utils/entityUtils.d.ts.map +0 -1
  249. package/dist/utils/entityUtils.js +0 -649
  250. package/dist/utils/entityUtils.js.map +0 -1
  251. package/dist/utils/errors.d.ts +0 -95
  252. package/dist/utils/errors.d.ts.map +0 -1
  253. package/dist/utils/errors.js +0 -146
  254. package/dist/utils/errors.js.map +0 -1
  255. package/dist/utils/formatters.d.ts +0 -145
  256. package/dist/utils/formatters.d.ts.map +0 -1
  257. package/dist/utils/formatters.js +0 -133
  258. package/dist/utils/formatters.js.map +0 -1
  259. package/dist/utils/index.d.ts +0 -26
  260. package/dist/utils/index.d.ts.map +0 -1
  261. package/dist/utils/index.js +0 -88
  262. package/dist/utils/index.js.map +0 -1
  263. package/dist/utils/indexes.d.ts +0 -270
  264. package/dist/utils/indexes.d.ts.map +0 -1
  265. package/dist/utils/indexes.js +0 -527
  266. package/dist/utils/indexes.js.map +0 -1
  267. package/dist/utils/logger.d.ts +0 -31
  268. package/dist/utils/logger.d.ts.map +0 -1
  269. package/dist/utils/logger.js +0 -41
  270. package/dist/utils/logger.js.map +0 -1
  271. package/dist/utils/operationUtils.d.ts +0 -124
  272. package/dist/utils/operationUtils.d.ts.map +0 -1
  273. package/dist/utils/operationUtils.js +0 -176
  274. package/dist/utils/operationUtils.js.map +0 -1
  275. package/dist/utils/parallelUtils.d.ts +0 -76
  276. package/dist/utils/parallelUtils.d.ts.map +0 -1
  277. package/dist/utils/parallelUtils.js +0 -192
  278. package/dist/utils/parallelUtils.js.map +0 -1
  279. package/dist/utils/schemas.d.ts +0 -556
  280. package/dist/utils/schemas.d.ts.map +0 -1
  281. package/dist/utils/schemas.js +0 -485
  282. package/dist/utils/schemas.js.map +0 -1
  283. package/dist/utils/searchAlgorithms.d.ts +0 -99
  284. package/dist/utils/searchAlgorithms.d.ts.map +0 -1
  285. package/dist/utils/searchAlgorithms.js +0 -168
  286. package/dist/utils/searchAlgorithms.js.map +0 -1
  287. package/dist/utils/searchCache.d.ts +0 -108
  288. package/dist/utils/searchCache.d.ts.map +0 -1
  289. package/dist/utils/searchCache.js +0 -210
  290. package/dist/utils/searchCache.js.map +0 -1
  291. package/dist/utils/taskScheduler.d.ts +0 -294
  292. package/dist/utils/taskScheduler.d.ts.map +0 -1
  293. package/dist/utils/taskScheduler.js +0 -487
  294. package/dist/utils/taskScheduler.js.map +0 -1
  295. package/dist/workers/index.d.ts +0 -12
  296. package/dist/workers/index.d.ts.map +0 -1
  297. package/dist/workers/index.js +0 -10
  298. package/dist/workers/index.js.map +0 -1
  299. package/dist/workers/levenshteinWorker.d.ts +0 -60
  300. package/dist/workers/levenshteinWorker.d.ts.map +0 -1
@@ -1,531 +0,0 @@
1
- /**
2
- * Embedding Service
3
- *
4
- * Phase 4 Sprint 10: Provides embedding abstractions for semantic search.
5
- * Phase 12 Sprint 5: Added query/document prefixes, l2Normalize, mode parameter,
6
- * and batch embedding with progress callback.
7
- * Supports multiple providers: OpenAI (cloud) and local (transformers.js).
8
- *
9
- * @module search/EmbeddingService
10
- */
11
- import { EMBEDDING_DEFAULTS, OPENAI_API_CONFIG, getEmbeddingConfig, } from '../utils/constants.js';
12
- /**
13
- * Phase 12 Sprint 5: Prefixes for query-optimized embedding encoding.
14
- *
15
- * These prefixes are used to distinguish between query and document embeddings,
16
- * which can improve retrieval performance with asymmetric embedding models.
17
- */
18
- export const QUERY_PREFIX = 'query: ';
19
- export const DOCUMENT_PREFIX = 'passage: ';
20
- /**
21
- * Phase 12 Sprint 5: L2 normalize a vector for cosine similarity.
22
- *
23
- * Normalizes a vector to unit length (magnitude 1), which is required
24
- * for accurate cosine similarity calculations.
25
- *
26
- * @param vector - Input vector to normalize
27
- * @returns L2 normalized vector
28
- *
29
- * @example
30
- * ```typescript
31
- * const normalized = l2Normalize([3, 4]); // [0.6, 0.8]
32
- * ```
33
- */
34
- export function l2Normalize(vector) {
35
- let magnitude = 0;
36
- for (const v of vector) {
37
- magnitude += v * v;
38
- }
39
- magnitude = Math.sqrt(magnitude);
40
- if (magnitude === 0 || magnitude === 1) {
41
- return magnitude === 0 ? vector : vector.slice();
42
- }
43
- return vector.map(v => v / magnitude);
44
- }
45
- /**
46
- * OpenAI Embedding Service
47
- *
48
- * Uses OpenAI's text-embedding-3-small model for generating embeddings.
49
- * Supports single and batch embedding with rate limit handling.
50
- * Phase 12 Sprint 5: Added mode parameter and progress callback support.
51
- *
52
- * @example
53
- * ```typescript
54
- * const service = new OpenAIEmbeddingService('sk-...');
55
- * const embedding = await service.embed("Hello world", 'query');
56
- * console.log(`Generated ${embedding.length} dimensions`);
57
- * ```
58
- */
59
- export class OpenAIEmbeddingService {
60
- dimensions;
61
- provider = 'openai';
62
- model;
63
- apiKey;
64
- /**
65
- * Create an OpenAI embedding service.
66
- *
67
- * @param apiKey - OpenAI API key
68
- * @param model - Optional model override (default: text-embedding-3-small)
69
- */
70
- constructor(apiKey, model) {
71
- if (!apiKey) {
72
- throw new Error('OpenAI API key is required');
73
- }
74
- this.apiKey = apiKey;
75
- this.model = model || EMBEDDING_DEFAULTS.OPENAI_MODEL;
76
- this.dimensions = EMBEDDING_DEFAULTS.OPENAI_DIMENSIONS;
77
- }
78
- /**
79
- * Check if the service is ready.
80
- */
81
- async isReady() {
82
- return !!this.apiKey;
83
- }
84
- /**
85
- * Apply prefix to text based on embedding mode.
86
- *
87
- * @param text - Original text
88
- * @param mode - Embedding mode ('query' or 'document')
89
- * @returns Text with appropriate prefix
90
- */
91
- applyPrefix(text, mode = 'document') {
92
- return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
93
- }
94
- /**
95
- * Generate embedding for a single text.
96
- *
97
- * @param text - Text to embed
98
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
99
- * @returns Embedding vector
100
- */
101
- async embed(text, mode = 'document') {
102
- const results = await this.embedBatch([text], mode);
103
- return results[0];
104
- }
105
- /**
106
- * Generate embeddings for multiple texts in batch.
107
- *
108
- * @param texts - Array of texts to embed
109
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
110
- * @returns Array of embedding vectors
111
- */
112
- async embedBatch(texts, mode = 'document') {
113
- if (texts.length === 0) {
114
- return [];
115
- }
116
- // Apply prefix based on mode
117
- const prefixedTexts = texts.map(text => this.applyPrefix(text, mode));
118
- // Split into batches if needed
119
- const maxBatchSize = EMBEDDING_DEFAULTS.OPENAI_MAX_BATCH_SIZE;
120
- const results = [];
121
- for (let i = 0; i < prefixedTexts.length; i += maxBatchSize) {
122
- const batch = prefixedTexts.slice(i, i + maxBatchSize);
123
- const batchResults = await this.embedBatchInternal(batch);
124
- results.push(...batchResults);
125
- }
126
- return results;
127
- }
128
- /**
129
- * Generate embeddings with progress callback support.
130
- *
131
- * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
132
- *
133
- * @param texts - Array of texts to embed
134
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
135
- * @param onProgress - Optional progress callback
136
- * @returns Array of embedding vectors
137
- */
138
- async embedBatchWithProgress(texts, mode = 'document', onProgress) {
139
- if (texts.length === 0) {
140
- return [];
141
- }
142
- // Apply prefix based on mode
143
- const prefixedTexts = texts.map(text => this.applyPrefix(text, mode));
144
- // Split into batches if needed
145
- const maxBatchSize = EMBEDDING_DEFAULTS.OPENAI_MAX_BATCH_SIZE;
146
- const results = [];
147
- const total = prefixedTexts.length;
148
- for (let i = 0; i < prefixedTexts.length; i += maxBatchSize) {
149
- const batch = prefixedTexts.slice(i, i + maxBatchSize);
150
- const batchResults = await this.embedBatchInternal(batch);
151
- results.push(...batchResults);
152
- // Report progress
153
- if (onProgress) {
154
- const current = Math.min(i + maxBatchSize, total);
155
- onProgress({
156
- current,
157
- total,
158
- percentage: Math.round((current / total) * 100),
159
- });
160
- }
161
- }
162
- return results;
163
- }
164
- /**
165
- * Internal batch embedding with retry logic.
166
- */
167
- async embedBatchInternal(texts) {
168
- let lastError = null;
169
- let backoff = OPENAI_API_CONFIG.INITIAL_BACKOFF_MS;
170
- for (let attempt = 0; attempt <= OPENAI_API_CONFIG.MAX_RETRIES; attempt++) {
171
- try {
172
- const response = await fetch(`${OPENAI_API_CONFIG.BASE_URL}${OPENAI_API_CONFIG.EMBEDDINGS_ENDPOINT}`, {
173
- method: 'POST',
174
- headers: {
175
- 'Content-Type': 'application/json',
176
- 'Authorization': `Bearer ${this.apiKey}`,
177
- },
178
- body: JSON.stringify({
179
- model: this.model,
180
- input: texts,
181
- }),
182
- });
183
- if (!response.ok) {
184
- const errorBody = await response.text();
185
- // Handle rate limiting
186
- if (response.status === 429) {
187
- if (attempt < OPENAI_API_CONFIG.MAX_RETRIES) {
188
- await this.sleep(backoff);
189
- backoff = Math.min(backoff * 2, OPENAI_API_CONFIG.MAX_BACKOFF_MS);
190
- continue;
191
- }
192
- }
193
- throw new Error(`OpenAI API error: ${response.status} - ${errorBody}`);
194
- }
195
- const data = await response.json();
196
- // Sort by index to ensure correct order
197
- const sortedData = [...data.data].sort((a, b) => a.index - b.index);
198
- return sortedData.map(item => item.embedding);
199
- }
200
- catch (error) {
201
- lastError = error instanceof Error ? error : new Error(String(error));
202
- // Retry on network errors
203
- if (attempt < OPENAI_API_CONFIG.MAX_RETRIES && this.isRetryableError(error)) {
204
- await this.sleep(backoff);
205
- backoff = Math.min(backoff * 2, OPENAI_API_CONFIG.MAX_BACKOFF_MS);
206
- continue;
207
- }
208
- throw lastError;
209
- }
210
- }
211
- throw lastError || new Error('Failed to generate embeddings after retries');
212
- }
213
- /**
214
- * Check if an error is retryable.
215
- */
216
- isRetryableError(error) {
217
- if (error instanceof Error) {
218
- // Network errors and rate limits are retryable
219
- return error.message.includes('fetch') ||
220
- error.message.includes('network') ||
221
- error.message.includes('429');
222
- }
223
- return false;
224
- }
225
- /**
226
- * Sleep for a given duration.
227
- */
228
- sleep(ms) {
229
- return new Promise(resolve => setTimeout(resolve, ms));
230
- }
231
- }
232
- /**
233
- * Local Embedding Service
234
- *
235
- * Uses @xenova/transformers for local embedding generation.
236
- * No API calls needed - runs entirely offline after initial model download.
237
- * Phase 12 Sprint 5: Added mode parameter and progress callback support.
238
- *
239
- * Note: Requires @xenova/transformers to be installed as an optional dependency.
240
- * If not available, initialization will fail gracefully.
241
- *
242
- * @example
243
- * ```typescript
244
- * const service = new LocalEmbeddingService();
245
- * await service.initialize();
246
- * const embedding = await service.embed("Hello world", 'query');
247
- * ```
248
- */
249
- export class LocalEmbeddingService {
250
- dimensions = EMBEDDING_DEFAULTS.LOCAL_DIMENSIONS;
251
- provider = 'local';
252
- model;
253
- pipeline = null;
254
- initialized = false;
255
- initPromise = null;
256
- /**
257
- * Create a local embedding service.
258
- *
259
- * @param model - Optional model override (default: Xenova/all-MiniLM-L6-v2)
260
- */
261
- constructor(model) {
262
- this.model = model || EMBEDDING_DEFAULTS.LOCAL_MODEL;
263
- }
264
- /**
265
- * Initialize the model pipeline.
266
- * Must be called before using embed/embedBatch.
267
- */
268
- async initialize() {
269
- if (this.initialized)
270
- return;
271
- if (this.initPromise) {
272
- return this.initPromise;
273
- }
274
- this.initPromise = this.initializeInternal();
275
- return this.initPromise;
276
- }
277
- /**
278
- * Internal initialization.
279
- */
280
- async initializeInternal() {
281
- try {
282
- // Dynamic import to allow optional dependency
283
- // @ts-expect-error - @xenova/transformers is an optional peer dependency
284
- const transformers = await import('@xenova/transformers');
285
- const { pipeline } = transformers;
286
- this.pipeline = await pipeline('feature-extraction', this.model);
287
- this.initialized = true;
288
- }
289
- catch (error) {
290
- this.initPromise = null;
291
- throw new Error(`Failed to initialize local embedding service: ${error instanceof Error ? error.message : String(error)}. ` +
292
- 'Make sure @xenova/transformers is installed.');
293
- }
294
- }
295
- /**
296
- * Check if the service is ready.
297
- */
298
- async isReady() {
299
- if (!this.initialized && !this.initPromise) {
300
- try {
301
- await this.initialize();
302
- }
303
- catch {
304
- return false;
305
- }
306
- }
307
- return this.initialized;
308
- }
309
- /**
310
- * Apply prefix to text based on embedding mode.
311
- *
312
- * @param text - Original text
313
- * @param mode - Embedding mode ('query' or 'document')
314
- * @returns Text with appropriate prefix
315
- */
316
- applyPrefix(text, mode = 'document') {
317
- return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
318
- }
319
- /**
320
- * Generate embedding for a single text.
321
- *
322
- * @param text - Text to embed
323
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
324
- * @returns Embedding vector
325
- */
326
- async embed(text, mode = 'document') {
327
- await this.ensureInitialized();
328
- const prefixedText = this.applyPrefix(text, mode);
329
- const pipelineFn = this.pipeline;
330
- const output = await pipelineFn(prefixedText, { pooling: 'mean', normalize: true });
331
- return Array.from(output.data);
332
- }
333
- /**
334
- * Generate embeddings for multiple texts in batch.
335
- * Note: Local processing is done sequentially to avoid memory issues.
336
- *
337
- * @param texts - Array of texts to embed
338
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
339
- * @returns Array of embedding vectors
340
- */
341
- async embedBatch(texts, mode = 'document') {
342
- await this.ensureInitialized();
343
- const results = [];
344
- for (const text of texts) {
345
- const embedding = await this.embed(text, mode);
346
- results.push(embedding);
347
- }
348
- return results;
349
- }
350
- /**
351
- * Generate embeddings with progress callback support.
352
- *
353
- * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
354
- *
355
- * @param texts - Array of texts to embed
356
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
357
- * @param onProgress - Optional progress callback
358
- * @returns Array of embedding vectors
359
- */
360
- async embedBatchWithProgress(texts, mode = 'document', onProgress) {
361
- await this.ensureInitialized();
362
- const results = [];
363
- const total = texts.length;
364
- for (let i = 0; i < texts.length; i++) {
365
- const embedding = await this.embed(texts[i], mode);
366
- results.push(embedding);
367
- // Report progress
368
- if (onProgress) {
369
- const current = i + 1;
370
- onProgress({
371
- current,
372
- total,
373
- percentage: Math.round((current / total) * 100),
374
- });
375
- }
376
- }
377
- return results;
378
- }
379
- /**
380
- * Ensure the service is initialized.
381
- */
382
- async ensureInitialized() {
383
- if (!this.initialized) {
384
- await this.initialize();
385
- }
386
- }
387
- }
388
- /**
389
- * Mock Embedding Service for testing
390
- *
391
- * Generates deterministic mock embeddings for testing purposes.
392
- * Useful for unit tests that don't need real embeddings.
393
- * Phase 12 Sprint 5: Added mode parameter and progress callback support.
394
- */
395
- export class MockEmbeddingService {
396
- dimensions;
397
- provider = 'mock';
398
- model = 'mock-model';
399
- /**
400
- * Create a mock embedding service.
401
- *
402
- * @param dimensions - Number of dimensions for mock embeddings
403
- */
404
- constructor(dimensions = 384) {
405
- this.dimensions = dimensions;
406
- }
407
- /**
408
- * Check if the service is ready.
409
- */
410
- async isReady() {
411
- return true;
412
- }
413
- /**
414
- * Apply prefix to text based on embedding mode.
415
- *
416
- * @param text - Original text
417
- * @param mode - Embedding mode ('query' or 'document')
418
- * @returns Text with appropriate prefix
419
- */
420
- applyPrefix(text, mode = 'document') {
421
- return mode === 'query' ? `${QUERY_PREFIX}${text}` : `${DOCUMENT_PREFIX}${text}`;
422
- }
423
- /**
424
- * Generate a deterministic mock embedding for a text.
425
- *
426
- * @param text - Text to embed
427
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
428
- * @returns Mock embedding vector
429
- */
430
- async embed(text, mode = 'document') {
431
- // Apply prefix based on mode (affects hash for different embeddings per mode)
432
- const prefixedText = this.applyPrefix(text, mode);
433
- // Generate deterministic embedding based on text hash
434
- const hash = this.hashString(prefixedText);
435
- const embedding = [];
436
- for (let i = 0; i < this.dimensions; i++) {
437
- // Use hash and index to generate deterministic values
438
- const value = Math.sin(hash + i * 0.1) * 0.5;
439
- embedding.push(value);
440
- }
441
- // Normalize the vector
442
- return this.normalize(embedding);
443
- }
444
- /**
445
- * Generate mock embeddings for multiple texts.
446
- *
447
- * @param texts - Array of texts to embed
448
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
449
- * @returns Array of mock embedding vectors
450
- */
451
- async embedBatch(texts, mode = 'document') {
452
- return Promise.all(texts.map(text => this.embed(text, mode)));
453
- }
454
- /**
455
- * Generate mock embeddings with progress callback support.
456
- *
457
- * Phase 12 Sprint 5: Added for tracking progress on large batch operations.
458
- *
459
- * @param texts - Array of texts to embed
460
- * @param mode - Embedding mode ('query' or 'document', default: 'document')
461
- * @param onProgress - Optional progress callback
462
- * @returns Array of mock embedding vectors
463
- */
464
- async embedBatchWithProgress(texts, mode = 'document', onProgress) {
465
- const results = [];
466
- const total = texts.length;
467
- for (let i = 0; i < texts.length; i++) {
468
- const embedding = await this.embed(texts[i], mode);
469
- results.push(embedding);
470
- // Report progress
471
- if (onProgress) {
472
- const current = i + 1;
473
- onProgress({
474
- current,
475
- total,
476
- percentage: Math.round((current / total) * 100),
477
- });
478
- }
479
- }
480
- return results;
481
- }
482
- /**
483
- * Simple string hash function.
484
- */
485
- hashString(str) {
486
- let hash = 0;
487
- for (let i = 0; i < str.length; i++) {
488
- const char = str.charCodeAt(i);
489
- hash = ((hash << 5) - hash) + char;
490
- hash = hash & hash; // Convert to 32bit integer
491
- }
492
- return hash;
493
- }
494
- /**
495
- * Normalize a vector to unit length.
496
- */
497
- normalize(vector) {
498
- let magnitude = 0;
499
- for (const v of vector) {
500
- magnitude += v * v;
501
- }
502
- magnitude = Math.sqrt(magnitude);
503
- if (magnitude === 0) {
504
- return vector;
505
- }
506
- return vector.map(v => v / magnitude);
507
- }
508
- }
509
- /**
510
- * Create an embedding service based on configuration.
511
- *
512
- * @param config - Optional configuration override
513
- * @returns Embedding service instance, or null if provider is 'none'
514
- */
515
- export function createEmbeddingService(config) {
516
- const envConfig = getEmbeddingConfig();
517
- const mergedConfig = { ...envConfig, ...config };
518
- switch (mergedConfig.provider) {
519
- case 'openai':
520
- if (!mergedConfig.apiKey) {
521
- throw new Error('OpenAI API key is required. Set MEMORY_OPENAI_API_KEY environment variable or provide apiKey in config.');
522
- }
523
- return new OpenAIEmbeddingService(mergedConfig.apiKey, mergedConfig.model);
524
- case 'local':
525
- return new LocalEmbeddingService(mergedConfig.model);
526
- case 'none':
527
- default:
528
- return null;
529
- }
530
- }
531
- //# sourceMappingURL=EmbeddingService.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"EmbeddingService.js","sourceRoot":"","sources":["../../src/search/EmbeddingService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,SAAS,CAAC;AACtC,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC;AAW3C;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,MAAgB;IAC1C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEjC,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,sBAAsB;IACxB,UAAU,CAAS;IACnB,QAAQ,GAAG,QAAQ,CAAC;IACpB,KAAK,CAAS;IACf,MAAM,CAAS;IAEvB;;;;;OAKG;IACH,YAAY,MAAc,EAAE,KAAc;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,kBAAkB,CAAC,YAAY,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,6BAA6B;QAC7B,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEtE,+BAA+B;QAC/B,MAAM,YAAY,GAAG,kBAAkB,CAAC,qBAAqB,CAAC;QAC9D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC;YACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,6BAA6B;QAC7B,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEtE,+BAA+B;QAC/B,MAAM,YAAY,GAAG,kBAAkB,CAAC,qBAAqB,CAAC;QAC9D,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC;YACvD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAE9B,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,CAAC;gBAClD,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAAC,KAAe;QAC9C,IAAI,SAAS,GAAiB,IAAI,CAAC;QACnC,IAAI,OAAO,GAAG,iBAAiB,CAAC,kBAAkB,CAAC;QAEnD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,GAAG,iBAAiB,CAAC,QAAQ,GAAG,iBAAiB,CAAC,mBAAmB,EAAE,EACvE;oBACE,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACzC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,KAAK,EAAE,KAAK;qBACb,CAAC;iBACH,CACF,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAExC,uBAAuB;oBACvB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC5B,IAAI,OAAO,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;4BAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;4BAC1B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;4BAClE,SAAS;wBACX,CAAC;oBACH,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;gBACzE,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;gBAE9D,wCAAwC;gBACxC,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBACpE,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,0BAA0B;gBAC1B,IAAI,OAAO,GAAG,iBAAiB,CAAC,WAAW,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC5E,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC1B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,MAAM,SAAS,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,+CAA+C;YAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAC/B,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACjC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF;AAmBD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,qBAAqB;IACvB,UAAU,GAAW,kBAAkB,CAAC,gBAAgB,CAAC;IACzD,QAAQ,GAAG,OAAO,CAAC;IACnB,KAAK,CAAS;IAEf,QAAQ,GAAY,IAAI,CAAC;IACzB,WAAW,GAAG,KAAK,CAAC;IACpB,WAAW,GAAyB,IAAI,CAAC;IAEjD;;;;OAIG;IACH,YAAY,KAAc;QACxB,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,kBAAkB,CAAC,WAAW,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC;YACH,8CAA8C;YAC9C,yEAAyE;YACzE,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC1D,MAAM,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC;YAElC,IAAI,CAAC,QAAQ,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,iDAAiD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBAC3G,8CAA8C,CAC/C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,QAA+G,CAAC;QACxI,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExB,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtB,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,oBAAoB;IACtB,UAAU,CAAS;IACnB,QAAQ,GAAG,MAAM,CAAC;IAClB,KAAK,GAAG,YAAY,CAAC;IAE9B;;;;OAIG;IACH,YAAY,aAAqB,GAAG;QAClC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAY,EAAE,OAAsB,UAAU;QAChE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,GAAG,IAAI,EAAE,CAAC;IACnF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAsB,UAAU;QACxD,8EAA8E;QAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAElD,sDAAsD;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,sDAAsD;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QAED,uBAAuB;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,OAAsB,UAAU;QAChE,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,sBAAsB,CAC1B,KAAe,EACf,OAAsB,UAAU,EAChC,UAAsC;QAEtC,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExB,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtB,UAAU,CAAC;oBACT,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;QACjD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,MAAgB;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IACxC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAiC;IACtE,MAAM,SAAS,GAAG,kBAAkB,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,EAAE,GAAG,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;IAEjD,QAAQ,YAAY,CAAC,QAAQ,EAAE,CAAC;QAC9B,KAAK,QAAQ;YACX,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CACb,yGAAyG,CAC1G,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,sBAAsB,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAE7E,KAAK,OAAO;YACV,OAAO,IAAI,qBAAqB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAEvD,KAAK,MAAM,CAAC;QACZ;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
@@ -1,118 +0,0 @@
1
- /**
2
- * Fuzzy Search
3
- *
4
- * Search with typo tolerance using Levenshtein distance similarity.
5
- * Uses workerpool for parallel processing on large datasets.
6
- *
7
- * @module search/FuzzySearch
8
- */
9
- import type { KnowledgeGraph } from '../types/index.js';
10
- import type { GraphStorage } from '../core/GraphStorage.js';
11
- /**
12
- * Default fuzzy search similarity threshold (70% match required).
13
- * Lower values are more permissive (more typos tolerated).
14
- * Higher values are stricter (fewer typos tolerated).
15
- */
16
- export declare const DEFAULT_FUZZY_THRESHOLD = 0.7;
17
- /**
18
- * Options for FuzzySearch constructor.
19
- */
20
- export interface FuzzySearchOptions {
21
- /**
22
- * Whether to use worker pool for parallel processing.
23
- * Set to false for testing or when workers are not available.
24
- * Default: true
25
- */
26
- useWorkerPool?: boolean;
27
- }
28
- /**
29
- * Performs fuzzy search with configurable similarity threshold.
30
- */
31
- export declare class FuzzySearch {
32
- private storage;
33
- /**
34
- * Phase 4 Sprint 3: Result cache for fuzzy search.
35
- * Maps cache key -> cached entity names.
36
- */
37
- private fuzzyResultCache;
38
- /**
39
- * Phase 8: Worker pool using workerpool library.
40
- * Initialized lazily when needed.
41
- */
42
- private workerPool;
43
- /**
44
- * Phase 7 Sprint 3: Path to the worker script.
45
- */
46
- private workerPath;
47
- /**
48
- * Phase 8: Whether to use worker pool for parallel processing.
49
- * Can be disabled for testing or when workers are not available.
50
- */
51
- private useWorkerPool;
52
- constructor(storage: GraphStorage, options?: FuzzySearchOptions);
53
- /**
54
- * Phase 4 Sprint 3: Generate cache key for fuzzy search parameters.
55
- */
56
- private generateCacheKey;
57
- /**
58
- * Phase 4 Sprint 3: Clear the fuzzy search cache.
59
- */
60
- clearCache(): void;
61
- /**
62
- * Phase 4 Sprint 3: Invalidate stale cache entries.
63
- */
64
- private cleanupCache;
65
- /**
66
- * Fuzzy search for entities with typo tolerance and pagination.
67
- *
68
- * Uses Levenshtein distance to calculate similarity between strings.
69
- * Matches if similarity >= threshold (0.0 to 1.0).
70
- *
71
- * Phase 4 Sprint 3: Implements result caching for repeated queries.
72
- *
73
- * @param query - Search query
74
- * @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_FUZZY_THRESHOLD
75
- * @param tags - Optional tags filter
76
- * @param minImportance - Optional minimum importance
77
- * @param maxImportance - Optional maximum importance
78
- * @param offset - Number of results to skip (default: 0)
79
- * @param limit - Maximum number of results (default: 50, max: 200)
80
- * @returns Filtered knowledge graph with fuzzy matches and pagination applied
81
- */
82
- fuzzySearch(query: string, threshold?: number, tags?: string[], minImportance?: number, maxImportance?: number, offset?: number, limit?: number): Promise<KnowledgeGraph>;
83
- /**
84
- * Phase 4 Sprint 3: Perform the actual fuzzy matching logic.
85
- * Extracted from fuzzySearch for cleaner code structure.
86
- */
87
- private performFuzzyMatch;
88
- /**
89
- * Check if two already-lowercase strings match with fuzzy logic.
90
- *
91
- * OPTIMIZED: Skips toLowerCase() calls when strings are already lowercase.
92
- *
93
- * @param s1 - First string (already lowercase)
94
- * @param s2 - Second string (already lowercase)
95
- * @param threshold - Similarity threshold (0.0 to 1.0)
96
- * @returns True if strings match fuzzily
97
- */
98
- private isFuzzyMatchLower;
99
- /**
100
- * Phase 8: Perform fuzzy search using workerpool for parallel processing.
101
- *
102
- * Splits entities into chunks and processes them in parallel using worker threads.
103
- * Falls back to single-threaded search if worker execution fails.
104
- *
105
- * @param query - Search query
106
- * @param threshold - Similarity threshold
107
- * @param entities - Entities to search
108
- * @returns Array of matched entities
109
- */
110
- private searchWithWorkers;
111
- /**
112
- * Phase 8: Shutdown the worker pool and clean up resources.
113
- *
114
- * Should be called when FuzzySearch is no longer needed.
115
- */
116
- shutdown(): Promise<void>;
117
- }
118
- //# sourceMappingURL=FuzzySearch.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"FuzzySearch.d.ts","sourceRoot":"","sources":["../../src/search/FuzzySearch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAU,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAQ5D;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AA4C3C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,qBAAa,WAAW;IAwBV,OAAO,CAAC,OAAO;IAvB3B;;;OAGG;IACH,OAAO,CAAC,gBAAgB,CAA2C;IAEnE;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAqB;IAEvC;;OAEG;IACH,OAAO,CAAC,UAAU,CAAS;IAE3B;;;OAGG;IACH,OAAO,CAAC,aAAa,CAAU;gBAEX,OAAO,EAAE,YAAY,EAAE,OAAO,GAAE,kBAAuB;IAmB3E;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAoBxB;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAwBpB;;;;;;;;;;;;;;;;OAgBG;IACG,WAAW,CACf,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAgC,EAC3C,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,aAAa,CAAC,EAAE,MAAM,EACtB,aAAa,CAAC,EAAE,MAAM,EACtB,MAAM,GAAE,MAAU,EAClB,KAAK,GAAE,MAA8B,GACpC,OAAO,CAAC,cAAc,CAAC;IAsE1B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA0BzB;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;IAezB;;;;;;;;;;OAUG;YACW,iBAAiB;IAiE/B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAMhC"}