@danielsimonjr/memoryjs 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/README.md +385 -113
  2. package/README.md.backup-1768084780988 +266 -0
  3. package/dist/index.cjs +24156 -0
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.cts +16967 -0
  6. package/dist/index.d.ts +16963 -11
  7. package/dist/index.js +23887 -19
  8. package/dist/index.js.map +1 -1
  9. package/dist/workers/levenshteinWorker.cjs +102 -0
  10. package/dist/workers/levenshteinWorker.cjs.map +1 -0
  11. package/dist/workers/levenshteinWorker.js +57 -91
  12. package/dist/workers/levenshteinWorker.js.map +1 -1
  13. package/package.json +75 -69
  14. package/dist/core/EntityManager.d.ts +0 -268
  15. package/dist/core/EntityManager.d.ts.map +0 -1
  16. package/dist/core/EntityManager.js +0 -512
  17. package/dist/core/EntityManager.js.map +0 -1
  18. package/dist/core/GraphEventEmitter.d.ts +0 -202
  19. package/dist/core/GraphEventEmitter.d.ts.map +0 -1
  20. package/dist/core/GraphEventEmitter.js +0 -347
  21. package/dist/core/GraphEventEmitter.js.map +0 -1
  22. package/dist/core/GraphStorage.d.ts +0 -395
  23. package/dist/core/GraphStorage.d.ts.map +0 -1
  24. package/dist/core/GraphStorage.js +0 -786
  25. package/dist/core/GraphStorage.js.map +0 -1
  26. package/dist/core/GraphTraversal.d.ts +0 -141
  27. package/dist/core/GraphTraversal.d.ts.map +0 -1
  28. package/dist/core/GraphTraversal.js +0 -574
  29. package/dist/core/GraphTraversal.js.map +0 -1
  30. package/dist/core/HierarchyManager.d.ts +0 -111
  31. package/dist/core/HierarchyManager.d.ts.map +0 -1
  32. package/dist/core/HierarchyManager.js +0 -225
  33. package/dist/core/HierarchyManager.js.map +0 -1
  34. package/dist/core/ManagerContext.d.ts +0 -76
  35. package/dist/core/ManagerContext.d.ts.map +0 -1
  36. package/dist/core/ManagerContext.js +0 -129
  37. package/dist/core/ManagerContext.js.map +0 -1
  38. package/dist/core/ObservationManager.d.ts +0 -85
  39. package/dist/core/ObservationManager.d.ts.map +0 -1
  40. package/dist/core/ObservationManager.js +0 -124
  41. package/dist/core/ObservationManager.js.map +0 -1
  42. package/dist/core/RelationManager.d.ts +0 -131
  43. package/dist/core/RelationManager.d.ts.map +0 -1
  44. package/dist/core/RelationManager.js +0 -212
  45. package/dist/core/RelationManager.js.map +0 -1
  46. package/dist/core/SQLiteStorage.d.ts +0 -354
  47. package/dist/core/SQLiteStorage.d.ts.map +0 -1
  48. package/dist/core/SQLiteStorage.js +0 -919
  49. package/dist/core/SQLiteStorage.js.map +0 -1
  50. package/dist/core/StorageFactory.d.ts +0 -45
  51. package/dist/core/StorageFactory.d.ts.map +0 -1
  52. package/dist/core/StorageFactory.js +0 -65
  53. package/dist/core/StorageFactory.js.map +0 -1
  54. package/dist/core/TransactionManager.d.ts +0 -464
  55. package/dist/core/TransactionManager.d.ts.map +0 -1
  56. package/dist/core/TransactionManager.js +0 -869
  57. package/dist/core/TransactionManager.js.map +0 -1
  58. package/dist/core/index.d.ts +0 -17
  59. package/dist/core/index.d.ts.map +0 -1
  60. package/dist/core/index.js +0 -20
  61. package/dist/core/index.js.map +0 -1
  62. package/dist/features/AnalyticsManager.d.ts +0 -44
  63. package/dist/features/AnalyticsManager.d.ts.map +0 -1
  64. package/dist/features/AnalyticsManager.js +0 -224
  65. package/dist/features/AnalyticsManager.js.map +0 -1
  66. package/dist/features/ArchiveManager.d.ts +0 -133
  67. package/dist/features/ArchiveManager.d.ts.map +0 -1
  68. package/dist/features/ArchiveManager.js +0 -282
  69. package/dist/features/ArchiveManager.js.map +0 -1
  70. package/dist/features/CompressionManager.d.ts +0 -119
  71. package/dist/features/CompressionManager.d.ts.map +0 -1
  72. package/dist/features/CompressionManager.js +0 -470
  73. package/dist/features/CompressionManager.js.map +0 -1
  74. package/dist/features/IOManager.d.ts +0 -225
  75. package/dist/features/IOManager.d.ts.map +0 -1
  76. package/dist/features/IOManager.js +0 -1093
  77. package/dist/features/IOManager.js.map +0 -1
  78. package/dist/features/KeywordExtractor.d.ts +0 -61
  79. package/dist/features/KeywordExtractor.d.ts.map +0 -1
  80. package/dist/features/KeywordExtractor.js +0 -127
  81. package/dist/features/KeywordExtractor.js.map +0 -1
  82. package/dist/features/ObservationNormalizer.d.ts +0 -90
  83. package/dist/features/ObservationNormalizer.d.ts.map +0 -1
  84. package/dist/features/ObservationNormalizer.js +0 -194
  85. package/dist/features/ObservationNormalizer.js.map +0 -1
  86. package/dist/features/StreamingExporter.d.ts +0 -128
  87. package/dist/features/StreamingExporter.d.ts.map +0 -1
  88. package/dist/features/StreamingExporter.js +0 -212
  89. package/dist/features/StreamingExporter.js.map +0 -1
  90. package/dist/features/TagManager.d.ts +0 -147
  91. package/dist/features/TagManager.d.ts.map +0 -1
  92. package/dist/features/TagManager.js +0 -211
  93. package/dist/features/TagManager.js.map +0 -1
  94. package/dist/features/index.d.ts +0 -14
  95. package/dist/features/index.d.ts.map +0 -1
  96. package/dist/features/index.js +0 -15
  97. package/dist/features/index.js.map +0 -1
  98. package/dist/index.d.ts.map +0 -1
  99. package/dist/search/BM25Search.d.ts +0 -148
  100. package/dist/search/BM25Search.d.ts.map +0 -1
  101. package/dist/search/BM25Search.js +0 -340
  102. package/dist/search/BM25Search.js.map +0 -1
  103. package/dist/search/BasicSearch.d.ts +0 -51
  104. package/dist/search/BasicSearch.d.ts.map +0 -1
  105. package/dist/search/BasicSearch.js +0 -138
  106. package/dist/search/BasicSearch.js.map +0 -1
  107. package/dist/search/BooleanSearch.d.ts +0 -98
  108. package/dist/search/BooleanSearch.d.ts.map +0 -1
  109. package/dist/search/BooleanSearch.js +0 -431
  110. package/dist/search/BooleanSearch.js.map +0 -1
  111. package/dist/search/EarlyTerminationManager.d.ts +0 -140
  112. package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
  113. package/dist/search/EarlyTerminationManager.js +0 -280
  114. package/dist/search/EarlyTerminationManager.js.map +0 -1
  115. package/dist/search/EmbeddingCache.d.ts +0 -175
  116. package/dist/search/EmbeddingCache.d.ts.map +0 -1
  117. package/dist/search/EmbeddingCache.js +0 -247
  118. package/dist/search/EmbeddingCache.js.map +0 -1
  119. package/dist/search/EmbeddingService.d.ts +0 -277
  120. package/dist/search/EmbeddingService.d.ts.map +0 -1
  121. package/dist/search/EmbeddingService.js +0 -531
  122. package/dist/search/EmbeddingService.js.map +0 -1
  123. package/dist/search/FuzzySearch.d.ts +0 -118
  124. package/dist/search/FuzzySearch.d.ts.map +0 -1
  125. package/dist/search/FuzzySearch.js +0 -313
  126. package/dist/search/FuzzySearch.js.map +0 -1
  127. package/dist/search/HybridScorer.d.ts +0 -181
  128. package/dist/search/HybridScorer.d.ts.map +0 -1
  129. package/dist/search/HybridScorer.js +0 -258
  130. package/dist/search/HybridScorer.js.map +0 -1
  131. package/dist/search/HybridSearchManager.d.ts +0 -80
  132. package/dist/search/HybridSearchManager.d.ts.map +0 -1
  133. package/dist/search/HybridSearchManager.js +0 -188
  134. package/dist/search/HybridSearchManager.js.map +0 -1
  135. package/dist/search/IncrementalIndexer.d.ts +0 -201
  136. package/dist/search/IncrementalIndexer.d.ts.map +0 -1
  137. package/dist/search/IncrementalIndexer.js +0 -343
  138. package/dist/search/IncrementalIndexer.js.map +0 -1
  139. package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
  140. package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
  141. package/dist/search/OptimizedInvertedIndex.js +0 -359
  142. package/dist/search/OptimizedInvertedIndex.js.map +0 -1
  143. package/dist/search/ParallelSearchExecutor.d.ts +0 -172
  144. package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
  145. package/dist/search/ParallelSearchExecutor.js +0 -310
  146. package/dist/search/ParallelSearchExecutor.js.map +0 -1
  147. package/dist/search/QuantizedVectorStore.d.ts +0 -171
  148. package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
  149. package/dist/search/QuantizedVectorStore.js +0 -308
  150. package/dist/search/QuantizedVectorStore.js.map +0 -1
  151. package/dist/search/QueryAnalyzer.d.ts +0 -76
  152. package/dist/search/QueryAnalyzer.d.ts.map +0 -1
  153. package/dist/search/QueryAnalyzer.js +0 -228
  154. package/dist/search/QueryAnalyzer.js.map +0 -1
  155. package/dist/search/QueryCostEstimator.d.ts +0 -244
  156. package/dist/search/QueryCostEstimator.d.ts.map +0 -1
  157. package/dist/search/QueryCostEstimator.js +0 -653
  158. package/dist/search/QueryCostEstimator.js.map +0 -1
  159. package/dist/search/QueryPlanCache.d.ts +0 -220
  160. package/dist/search/QueryPlanCache.d.ts.map +0 -1
  161. package/dist/search/QueryPlanCache.js +0 -380
  162. package/dist/search/QueryPlanCache.js.map +0 -1
  163. package/dist/search/QueryPlanner.d.ts +0 -58
  164. package/dist/search/QueryPlanner.d.ts.map +0 -1
  165. package/dist/search/QueryPlanner.js +0 -138
  166. package/dist/search/QueryPlanner.js.map +0 -1
  167. package/dist/search/RankedSearch.d.ts +0 -71
  168. package/dist/search/RankedSearch.d.ts.map +0 -1
  169. package/dist/search/RankedSearch.js +0 -239
  170. package/dist/search/RankedSearch.js.map +0 -1
  171. package/dist/search/ReflectionManager.d.ts +0 -120
  172. package/dist/search/ReflectionManager.d.ts.map +0 -1
  173. package/dist/search/ReflectionManager.js +0 -232
  174. package/dist/search/ReflectionManager.js.map +0 -1
  175. package/dist/search/SavedSearchManager.d.ts +0 -79
  176. package/dist/search/SavedSearchManager.d.ts.map +0 -1
  177. package/dist/search/SavedSearchManager.js +0 -147
  178. package/dist/search/SavedSearchManager.js.map +0 -1
  179. package/dist/search/SearchFilterChain.d.ts +0 -120
  180. package/dist/search/SearchFilterChain.d.ts.map +0 -1
  181. package/dist/search/SearchFilterChain.js +0 -186
  182. package/dist/search/SearchFilterChain.js.map +0 -1
  183. package/dist/search/SearchManager.d.ts +0 -326
  184. package/dist/search/SearchManager.d.ts.map +0 -1
  185. package/dist/search/SearchManager.js +0 -454
  186. package/dist/search/SearchManager.js.map +0 -1
  187. package/dist/search/SearchSuggestions.d.ts +0 -27
  188. package/dist/search/SearchSuggestions.d.ts.map +0 -1
  189. package/dist/search/SearchSuggestions.js +0 -58
  190. package/dist/search/SearchSuggestions.js.map +0 -1
  191. package/dist/search/SemanticSearch.d.ts +0 -149
  192. package/dist/search/SemanticSearch.d.ts.map +0 -1
  193. package/dist/search/SemanticSearch.js +0 -324
  194. package/dist/search/SemanticSearch.js.map +0 -1
  195. package/dist/search/SymbolicSearch.d.ts +0 -61
  196. package/dist/search/SymbolicSearch.d.ts.map +0 -1
  197. package/dist/search/SymbolicSearch.js +0 -164
  198. package/dist/search/SymbolicSearch.js.map +0 -1
  199. package/dist/search/TFIDFEventSync.d.ts +0 -85
  200. package/dist/search/TFIDFEventSync.d.ts.map +0 -1
  201. package/dist/search/TFIDFEventSync.js +0 -134
  202. package/dist/search/TFIDFEventSync.js.map +0 -1
  203. package/dist/search/TFIDFIndexManager.d.ts +0 -151
  204. package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
  205. package/dist/search/TFIDFIndexManager.js +0 -433
  206. package/dist/search/TFIDFIndexManager.js.map +0 -1
  207. package/dist/search/VectorStore.d.ts +0 -235
  208. package/dist/search/VectorStore.d.ts.map +0 -1
  209. package/dist/search/VectorStore.js +0 -312
  210. package/dist/search/VectorStore.js.map +0 -1
  211. package/dist/search/index.d.ts +0 -35
  212. package/dist/search/index.d.ts.map +0 -1
  213. package/dist/search/index.js +0 -53
  214. package/dist/search/index.js.map +0 -1
  215. package/dist/types/index.d.ts +0 -13
  216. package/dist/types/index.d.ts.map +0 -1
  217. package/dist/types/index.js +0 -13
  218. package/dist/types/index.js.map +0 -1
  219. package/dist/types/types.d.ts +0 -1811
  220. package/dist/types/types.d.ts.map +0 -1
  221. package/dist/types/types.js +0 -10
  222. package/dist/types/types.js.map +0 -1
  223. package/dist/utils/BatchProcessor.d.ts +0 -271
  224. package/dist/utils/BatchProcessor.d.ts.map +0 -1
  225. package/dist/utils/BatchProcessor.js +0 -377
  226. package/dist/utils/BatchProcessor.js.map +0 -1
  227. package/dist/utils/MemoryMonitor.d.ts +0 -176
  228. package/dist/utils/MemoryMonitor.d.ts.map +0 -1
  229. package/dist/utils/MemoryMonitor.js +0 -306
  230. package/dist/utils/MemoryMonitor.js.map +0 -1
  231. package/dist/utils/WorkerPoolManager.d.ts +0 -233
  232. package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
  233. package/dist/utils/WorkerPoolManager.js +0 -421
  234. package/dist/utils/WorkerPoolManager.js.map +0 -1
  235. package/dist/utils/compressedCache.d.ts +0 -221
  236. package/dist/utils/compressedCache.d.ts.map +0 -1
  237. package/dist/utils/compressedCache.js +0 -349
  238. package/dist/utils/compressedCache.js.map +0 -1
  239. package/dist/utils/compressionUtil.d.ts +0 -214
  240. package/dist/utils/compressionUtil.d.ts.map +0 -1
  241. package/dist/utils/compressionUtil.js +0 -248
  242. package/dist/utils/compressionUtil.js.map +0 -1
  243. package/dist/utils/constants.d.ts +0 -245
  244. package/dist/utils/constants.d.ts.map +0 -1
  245. package/dist/utils/constants.js +0 -253
  246. package/dist/utils/constants.js.map +0 -1
  247. package/dist/utils/entityUtils.d.ts +0 -379
  248. package/dist/utils/entityUtils.d.ts.map +0 -1
  249. package/dist/utils/entityUtils.js +0 -649
  250. package/dist/utils/entityUtils.js.map +0 -1
  251. package/dist/utils/errors.d.ts +0 -95
  252. package/dist/utils/errors.d.ts.map +0 -1
  253. package/dist/utils/errors.js +0 -146
  254. package/dist/utils/errors.js.map +0 -1
  255. package/dist/utils/formatters.d.ts +0 -145
  256. package/dist/utils/formatters.d.ts.map +0 -1
  257. package/dist/utils/formatters.js +0 -133
  258. package/dist/utils/formatters.js.map +0 -1
  259. package/dist/utils/index.d.ts +0 -26
  260. package/dist/utils/index.d.ts.map +0 -1
  261. package/dist/utils/index.js +0 -88
  262. package/dist/utils/index.js.map +0 -1
  263. package/dist/utils/indexes.d.ts +0 -270
  264. package/dist/utils/indexes.d.ts.map +0 -1
  265. package/dist/utils/indexes.js +0 -527
  266. package/dist/utils/indexes.js.map +0 -1
  267. package/dist/utils/logger.d.ts +0 -31
  268. package/dist/utils/logger.d.ts.map +0 -1
  269. package/dist/utils/logger.js +0 -41
  270. package/dist/utils/logger.js.map +0 -1
  271. package/dist/utils/operationUtils.d.ts +0 -124
  272. package/dist/utils/operationUtils.d.ts.map +0 -1
  273. package/dist/utils/operationUtils.js +0 -176
  274. package/dist/utils/operationUtils.js.map +0 -1
  275. package/dist/utils/parallelUtils.d.ts +0 -76
  276. package/dist/utils/parallelUtils.d.ts.map +0 -1
  277. package/dist/utils/parallelUtils.js +0 -192
  278. package/dist/utils/parallelUtils.js.map +0 -1
  279. package/dist/utils/schemas.d.ts +0 -556
  280. package/dist/utils/schemas.d.ts.map +0 -1
  281. package/dist/utils/schemas.js +0 -485
  282. package/dist/utils/schemas.js.map +0 -1
  283. package/dist/utils/searchAlgorithms.d.ts +0 -99
  284. package/dist/utils/searchAlgorithms.d.ts.map +0 -1
  285. package/dist/utils/searchAlgorithms.js +0 -168
  286. package/dist/utils/searchAlgorithms.js.map +0 -1
  287. package/dist/utils/searchCache.d.ts +0 -108
  288. package/dist/utils/searchCache.d.ts.map +0 -1
  289. package/dist/utils/searchCache.js +0 -210
  290. package/dist/utils/searchCache.js.map +0 -1
  291. package/dist/utils/taskScheduler.d.ts +0 -294
  292. package/dist/utils/taskScheduler.d.ts.map +0 -1
  293. package/dist/utils/taskScheduler.js +0 -487
  294. package/dist/utils/taskScheduler.js.map +0 -1
  295. package/dist/workers/index.d.ts +0 -12
  296. package/dist/workers/index.d.ts.map +0 -1
  297. package/dist/workers/index.js +0 -10
  298. package/dist/workers/index.js.map +0 -1
  299. package/dist/workers/levenshteinWorker.d.ts +0 -60
  300. package/dist/workers/levenshteinWorker.d.ts.map +0 -1
@@ -1,433 +0,0 @@
1
- /**
2
- * TF-IDF Index Manager
3
- *
4
- * Manages pre-calculated TF-IDF indexes for fast ranked search.
5
- * Handles index building, incremental updates, and persistence.
6
- *
7
- * @module search/TFIDFIndexManager
8
- */
9
- import * as fs from 'fs/promises';
10
- import * as path from 'path';
11
- import { calculateIDFFromTokenSets, tokenize } from '../utils/index.js';
12
- const INDEX_VERSION = '1.0';
13
- const INDEX_FILENAME = 'tfidf-index.json';
14
- /**
15
- * Manages TF-IDF index lifecycle: building, updating, and persistence.
16
- */
17
- export class TFIDFIndexManager {
18
- indexPath;
19
- index = null;
20
- constructor(storageDir) {
21
- this.indexPath = path.join(storageDir, '.indexes', INDEX_FILENAME);
22
- }
23
- /**
24
- * Build a complete TF-IDF index from a knowledge graph.
25
- *
26
- * @param graph - Knowledge graph to index
27
- * @returns Newly built TF-IDF index
28
- */
29
- async buildIndex(graph) {
30
- const documents = new Map();
31
- const allTokenSets = [];
32
- // Build document vectors - tokenize once per document
33
- for (const entity of graph.entities) {
34
- const documentText = [
35
- entity.name,
36
- entity.entityType,
37
- ...entity.observations,
38
- ].join(' ');
39
- const tokens = tokenize(documentText);
40
- const tokenSet = new Set(tokens);
41
- allTokenSets.push(tokenSet);
42
- // Calculate term frequencies
43
- const termFreq = {};
44
- for (const term of tokens) {
45
- termFreq[term] = (termFreq[term] || 0) + 1;
46
- }
47
- documents.set(entity.name, {
48
- entityName: entity.name,
49
- terms: termFreq,
50
- documentText,
51
- });
52
- }
53
- // Calculate IDF for all terms using pre-tokenized sets (O(1) lookup per document)
54
- const idf = new Map();
55
- const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
56
- for (const term of allTerms) {
57
- const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
58
- idf.set(term, idfScore);
59
- }
60
- this.index = {
61
- version: INDEX_VERSION,
62
- lastUpdated: new Date().toISOString(),
63
- documents,
64
- idf,
65
- };
66
- return this.index;
67
- }
68
- /**
69
- * Update the index incrementally when entities change.
70
- *
71
- * More efficient than rebuilding the entire index.
72
- *
73
- * @param graph - Updated knowledge graph
74
- * @param changedEntityNames - Names of entities that changed
75
- */
76
- async updateIndex(graph, changedEntityNames) {
77
- if (!this.index) {
78
- // No existing index, build from scratch
79
- return this.buildIndex(graph);
80
- }
81
- // Rebuild document vectors for changed entities
82
- const allTokenSets = [];
83
- const updatedDocuments = new Map(this.index.documents);
84
- // Remove deleted entities
85
- for (const entityName of changedEntityNames) {
86
- const entity = graph.entities.find(e => e.name === entityName);
87
- if (!entity) {
88
- updatedDocuments.delete(entityName);
89
- }
90
- }
91
- // Update/add changed entities - tokenize once per document
92
- for (const entity of graph.entities) {
93
- const documentText = [
94
- entity.name,
95
- entity.entityType,
96
- ...entity.observations,
97
- ].join(' ');
98
- const tokens = tokenize(documentText);
99
- const tokenSet = new Set(tokens);
100
- allTokenSets.push(tokenSet);
101
- if (changedEntityNames.has(entity.name)) {
102
- // Calculate term frequencies for changed entity
103
- const termFreq = {};
104
- for (const term of tokens) {
105
- termFreq[term] = (termFreq[term] || 0) + 1;
106
- }
107
- updatedDocuments.set(entity.name, {
108
- entityName: entity.name,
109
- terms: termFreq,
110
- documentText,
111
- });
112
- }
113
- }
114
- // Recalculate IDF using pre-tokenized sets (O(1) lookup per document)
115
- const idf = new Map();
116
- const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
117
- for (const term of allTerms) {
118
- const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
119
- idf.set(term, idfScore);
120
- }
121
- this.index = {
122
- version: INDEX_VERSION,
123
- lastUpdated: new Date().toISOString(),
124
- documents: updatedDocuments,
125
- idf,
126
- };
127
- return this.index;
128
- }
129
- /**
130
- * Load index from disk.
131
- *
132
- * @returns Loaded index or null if not found
133
- */
134
- async loadIndex() {
135
- try {
136
- const data = await fs.readFile(this.indexPath, 'utf-8');
137
- const serialized = JSON.parse(data);
138
- this.index = {
139
- version: serialized.version,
140
- lastUpdated: serialized.lastUpdated,
141
- documents: new Map(serialized.documents),
142
- idf: new Map(serialized.idf),
143
- };
144
- return this.index;
145
- }
146
- catch (error) {
147
- // Index doesn't exist or is invalid
148
- return null;
149
- }
150
- }
151
- /**
152
- * Save index to disk.
153
- *
154
- * @param index - Index to save (uses cached index if not provided)
155
- */
156
- async saveIndex(index) {
157
- const indexToSave = index || this.index;
158
- if (!indexToSave) {
159
- throw new Error('No index to save');
160
- }
161
- // Ensure index directory exists
162
- const indexDir = path.dirname(this.indexPath);
163
- await fs.mkdir(indexDir, { recursive: true });
164
- // Serialize Map objects to arrays for JSON
165
- const serialized = {
166
- version: indexToSave.version,
167
- lastUpdated: indexToSave.lastUpdated,
168
- documents: Array.from(indexToSave.documents.entries()),
169
- idf: Array.from(indexToSave.idf.entries()),
170
- };
171
- await fs.writeFile(this.indexPath, JSON.stringify(serialized, null, 2), 'utf-8');
172
- }
173
- /**
174
- * Get the current cached index.
175
- *
176
- * @returns Cached index or null if not loaded
177
- */
178
- getIndex() {
179
- return this.index;
180
- }
181
- /**
182
- * Clear the cached index and delete from disk.
183
- */
184
- async clearIndex() {
185
- this.index = null;
186
- try {
187
- await fs.unlink(this.indexPath);
188
- }
189
- catch {
190
- // Index file doesn't exist, nothing to delete
191
- }
192
- }
193
- /**
194
- * Check if the index needs rebuilding based on graph state.
195
- *
196
- * @param graph - Current knowledge graph
197
- * @returns True if index should be rebuilt
198
- */
199
- needsRebuild(graph) {
200
- if (!this.index) {
201
- return true;
202
- }
203
- // Check if entity count matches
204
- if (this.index.documents.size !== graph.entities.length) {
205
- return true;
206
- }
207
- // Check if all entities are in index
208
- for (const entity of graph.entities) {
209
- if (!this.index.documents.has(entity.name)) {
210
- return true;
211
- }
212
- }
213
- return false;
214
- }
215
- // ==================== Phase 10 Sprint 3: Incremental Index Updates ====================
216
- /**
217
- * Phase 10 Sprint 3: Add a single document to the index incrementally.
218
- *
219
- * More efficient than rebuilding the entire index for single entity additions.
220
- * Updates TF for the new document and recalculates IDF for affected terms.
221
- *
222
- * @param entity - The entity to add
223
- *
224
- * @example
225
- * ```typescript
226
- * const indexManager = new TFIDFIndexManager('/data');
227
- * await indexManager.loadIndex();
228
- *
229
- * // Add new entity
230
- * indexManager.addDocument({
231
- * name: 'NewEntity',
232
- * entityType: 'person',
233
- * observations: ['Software engineer']
234
- * });
235
- * ```
236
- */
237
- addDocument(entity) {
238
- if (!this.index) {
239
- // Can't add to non-existent index
240
- return;
241
- }
242
- // Build document text and tokens
243
- const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
244
- const tokens = tokenize(documentText);
245
- // Calculate term frequencies
246
- const termFreq = {};
247
- for (const term of tokens) {
248
- termFreq[term] = (termFreq[term] || 0) + 1;
249
- }
250
- // Add to documents map
251
- this.index.documents.set(entity.name, {
252
- entityName: entity.name,
253
- terms: termFreq,
254
- documentText,
255
- });
256
- // Update IDF for ALL terms because N changed (total document count)
257
- // IDF = log(N/df), and N has increased
258
- this.recalculateAllIDF();
259
- // Update timestamp
260
- this.index.lastUpdated = new Date().toISOString();
261
- }
262
- /**
263
- * Phase 10 Sprint 3: Remove a single document from the index incrementally.
264
- *
265
- * More efficient than rebuilding the entire index for single entity deletions.
266
- * Recalculates IDF for terms that were in the removed document.
267
- *
268
- * @param entityName - Name of the entity to remove
269
- *
270
- * @example
271
- * ```typescript
272
- * indexManager.removeDocument('DeletedEntity');
273
- * ```
274
- */
275
- removeDocument(entityName) {
276
- if (!this.index) {
277
- return;
278
- }
279
- const document = this.index.documents.get(entityName);
280
- if (!document) {
281
- return;
282
- }
283
- // Remove from documents map
284
- this.index.documents.delete(entityName);
285
- // Update IDF for ALL terms because N changed (total document count)
286
- // IDF = log(N/df), and N has decreased
287
- this.recalculateAllIDF();
288
- // Update timestamp
289
- this.index.lastUpdated = new Date().toISOString();
290
- }
291
- /**
292
- * Phase 10 Sprint 3: Update a single document in the index incrementally.
293
- *
294
- * More efficient than rebuilding the entire index for single entity updates.
295
- * Handles both term changes and observation updates.
296
- *
297
- * @param entity - The updated entity
298
- *
299
- * @example
300
- * ```typescript
301
- * indexManager.updateDocument({
302
- * name: 'ExistingEntity',
303
- * entityType: 'person',
304
- * observations: ['Updated observations']
305
- * });
306
- * ```
307
- */
308
- updateDocument(entity) {
309
- if (!this.index) {
310
- return;
311
- }
312
- const oldDocument = this.index.documents.get(entity.name);
313
- const oldTerms = oldDocument ? new Set(Object.keys(oldDocument.terms)) : new Set();
314
- // Build new document
315
- const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
316
- const tokens = tokenize(documentText);
317
- const newTerms = new Set(tokens);
318
- // Calculate term frequencies
319
- const termFreq = {};
320
- for (const term of tokens) {
321
- termFreq[term] = (termFreq[term] || 0) + 1;
322
- }
323
- // Update documents map
324
- this.index.documents.set(entity.name, {
325
- entityName: entity.name,
326
- terms: termFreq,
327
- documentText,
328
- });
329
- // Find terms that changed (added or removed)
330
- const changedTerms = new Set();
331
- for (const term of oldTerms) {
332
- if (!newTerms.has(term)) {
333
- changedTerms.add(term);
334
- }
335
- }
336
- for (const term of newTerms) {
337
- if (!oldTerms.has(term)) {
338
- changedTerms.add(term);
339
- }
340
- }
341
- // Recalculate IDF for changed terms
342
- if (changedTerms.size > 0) {
343
- this.recalculateIDFForTerms(changedTerms);
344
- }
345
- // Update timestamp
346
- this.index.lastUpdated = new Date().toISOString();
347
- }
348
- /**
349
- * Phase 10 Sprint 3: Recalculate IDF scores for a set of terms.
350
- *
351
- * @param terms - Set of terms to recalculate IDF for
352
- * @private
353
- */
354
- recalculateIDFForTerms(terms) {
355
- if (!this.index) {
356
- return;
357
- }
358
- const totalDocs = this.index.documents.size;
359
- if (totalDocs === 0) {
360
- // No documents, clear all IDF for these terms
361
- for (const term of terms) {
362
- this.index.idf.delete(term);
363
- }
364
- return;
365
- }
366
- // Count documents containing each term
367
- for (const term of terms) {
368
- let docCount = 0;
369
- for (const doc of this.index.documents.values()) {
370
- if (term in doc.terms) {
371
- docCount++;
372
- }
373
- }
374
- if (docCount > 0) {
375
- // IDF = log(N / df) where N = total docs, df = doc frequency
376
- const idfScore = Math.log(totalDocs / docCount);
377
- this.index.idf.set(term, idfScore);
378
- }
379
- else {
380
- // Term no longer exists in any document
381
- this.index.idf.delete(term);
382
- }
383
- }
384
- }
385
- /**
386
- * Phase 10 Sprint 3: Recalculate IDF scores for ALL terms in the index.
387
- *
388
- * Called when the total document count changes (add/remove document).
389
- * @private
390
- */
391
- recalculateAllIDF() {
392
- if (!this.index) {
393
- return;
394
- }
395
- const totalDocs = this.index.documents.size;
396
- if (totalDocs === 0) {
397
- // No documents, clear all IDF
398
- this.index.idf.clear();
399
- return;
400
- }
401
- // Build term -> document count map
402
- const termDocCounts = new Map();
403
- for (const doc of this.index.documents.values()) {
404
- for (const term of Object.keys(doc.terms)) {
405
- termDocCounts.set(term, (termDocCounts.get(term) ?? 0) + 1);
406
- }
407
- }
408
- // Clear old IDF and recalculate
409
- this.index.idf.clear();
410
- for (const [term, docCount] of termDocCounts) {
411
- // IDF = log(N / df) where N = total docs, df = doc frequency
412
- const idfScore = Math.log(totalDocs / docCount);
413
- this.index.idf.set(term, idfScore);
414
- }
415
- }
416
- /**
417
- * Phase 10 Sprint 3: Check if the index is loaded/initialized.
418
- *
419
- * @returns True if index is available
420
- */
421
- isInitialized() {
422
- return this.index !== null;
423
- }
424
- /**
425
- * Phase 10 Sprint 3: Get the number of documents in the index.
426
- *
427
- * @returns Document count or 0 if not initialized
428
- */
429
- getDocumentCount() {
430
- return this.index?.documents.size ?? 0;
431
- }
432
- }
433
- //# sourceMappingURL=TFIDFIndexManager.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"TFIDFIndexManager.js","sourceRoot":"","sources":["../../src/search/TFIDFIndexManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAExE,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAY1C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,SAAS,CAAS;IAClB,KAAK,GAAsB,IAAI,CAAC;IAExC,YAAY,UAAkB;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;QACpD,MAAM,YAAY,GAAkB,EAAE,CAAC;QAEvC,sDAAsD;QACtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG;gBACnB,MAAM,CAAC,IAAI;gBACX,MAAM,CAAC,UAAU;gBACjB,GAAG,MAAM,CAAC,YAAY;aACvB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE5B,6BAA6B;YAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7C,CAAC;YAED,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;gBACzB,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,KAAK,EAAE,QAAQ;gBACf,YAAY;aACb,CAAC,CAAC;QACL,CAAC;QAED,kFAAkF;QAClF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAC/D,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,KAAK,GAAG;YACX,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS;YACT,GAAG;SACJ,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,KAA6B,EAAE,kBAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,wCAAwC;YACxC,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,gDAAgD;QAChD,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEvD,0BAA0B;QAC1B,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;YAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG;gBACnB,MAAM,CAAC,IAAI;gBACX,MAAM,CAAC,UAAU;gBACjB,GAAG,MAAM,CAAC,YAAY;aACvB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE5B,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,gDAAgD;gBAChD,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC7C,CAAC;gBAED,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;oBAChC,UAAU,EAAE,MAAM,CAAC,IAAI;oBACvB,KAAK,EAAE,QAAQ;oBACf,YAAY;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAC/D,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,KAAK,GAAG;YACX,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,gBAAgB;YAC3B,GAAG;SACJ,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,UAAU,GAAyB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE1D,IAAI,CAAC,KAAK,GAAG;gBACX,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,SAAS,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,GAAG,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;aAC7B,CAAC;YAEF,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,oCAAoC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS,CAAC,KAAkB;QAChC,MAAM,WAAW,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACxC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9C,2CAA2C;QAC3C,MAAM,UAAU,GAAyB;YACvC,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACtD,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;SAC3C,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACnF,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,8CAA8C;QAChD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,KAAqB;QAChC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qCAAqC;QACrC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3C,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yFAAyF;IAEzF;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,WAAW,CAAC,MAAoE;QAC9E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,kCAAkC;YAClC,OAAO;QACT,CAAC;QAED,iCAAiC;QACjC,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEtC,6BAA6B;QAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ;YACf,YAAY;SACb,CAAC,CAAC;QAEH,oEAAoE;QACpE,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,cAAc,CAAC,UAAkB;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAExC,oEAAoE;QACpE,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,MAAoE;QACjF,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAE3F,qBAAqB;QACrB,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAEjC,6BAA6B;QAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ;YACf,YAAY;SACb,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACK,sBAAsB,CAAC,KAAkB;QAC/C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,8CAA8C;YAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;gBAChD,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACtB,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,6DAA6D;gBAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,iBAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;QAE5C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,8BAA8B;YAC9B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YAC7C,6DAA6D;YAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC;IACzC,CAAC;CACF"}
@@ -1,235 +0,0 @@
1
- /**
2
- * Vector Store
3
- *
4
- * Phase 4 Sprint 11: Vector storage and retrieval for semantic search.
5
- * Provides in-memory and SQLite-backed implementations.
6
- *
7
- * @module search/VectorStore
8
- */
9
- import type { IVectorStore, VectorSearchResult } from '../types/index.js';
10
- /**
11
- * Calculate cosine similarity between two vectors.
12
- *
13
- * Uses an optimized inner loop without array methods for maximum performance.
14
- *
15
- * @param a - First vector
16
- * @param b - Second vector
17
- * @returns Cosine similarity score (0.0 to 1.0)
18
- */
19
- export declare function cosineSimilarity(a: number[], b: number[]): number;
20
- /**
21
- * In-Memory Vector Store
22
- *
23
- * Stores vectors in memory using a Map for O(1) add/remove operations.
24
- * Search uses brute-force cosine similarity which is O(n) but fast for
25
- * small to medium graphs (<10K entities).
26
- *
27
- * @example
28
- * ```typescript
29
- * const store = new InMemoryVectorStore();
30
- * store.add("entity1", [0.1, 0.2, 0.3]);
31
- * store.add("entity2", [0.4, 0.5, 0.6]);
32
- * const results = store.search([0.1, 0.2, 0.3], 5);
33
- * console.log(results); // [{ name: "entity1", score: 1.0 }, ...]
34
- * ```
35
- */
36
- export declare class InMemoryVectorStore implements IVectorStore {
37
- /** Map of entity name to embedding vector */
38
- private vectors;
39
- /**
40
- * Add a vector for an entity.
41
- *
42
- * @param entityName - Name of the entity
43
- * @param vector - Embedding vector
44
- */
45
- add(entityName: string, vector: number[]): void;
46
- /**
47
- * Search for similar vectors using cosine similarity.
48
- *
49
- * @param queryVector - Query embedding vector
50
- * @param k - Number of results to return
51
- * @returns Array of results with entity name and similarity score
52
- */
53
- search(queryVector: number[], k: number): VectorSearchResult[];
54
- /**
55
- * Remove a vector by entity name.
56
- *
57
- * @param entityName - Name of the entity to remove
58
- * @returns True if found and removed
59
- */
60
- remove(entityName: string): boolean;
61
- /**
62
- * Get the number of vectors stored.
63
- *
64
- * @returns Number of vectors
65
- */
66
- size(): number;
67
- /**
68
- * Clear all vectors from the store.
69
- */
70
- clear(): void;
71
- /**
72
- * Check if a vector exists for an entity.
73
- *
74
- * @param entityName - Name of the entity
75
- * @returns True if vector exists
76
- */
77
- has(entityName: string): boolean;
78
- /**
79
- * Get the vector for an entity.
80
- *
81
- * @param entityName - Name of the entity
82
- * @returns Vector if found, undefined otherwise
83
- */
84
- get(entityName: string): number[] | undefined;
85
- /**
86
- * Get all entity names with stored vectors.
87
- *
88
- * @returns Array of entity names
89
- */
90
- getEntityNames(): string[];
91
- /**
92
- * Load vectors from an iterable source.
93
- *
94
- * @param entries - Iterable of [entityName, vector] pairs
95
- */
96
- loadFrom(entries: Iterable<[string, number[]]>): void;
97
- }
98
- /**
99
- * SQLite Vector Store
100
- *
101
- * Persists vectors to SQLite storage while maintaining an in-memory cache
102
- * for fast search operations. Combines persistence with performance.
103
- *
104
- * Uses SQLiteStorage's embedding storage methods for persistence.
105
- *
106
- * @example
107
- * ```typescript
108
- * const store = new SQLiteVectorStore(sqliteStorage);
109
- * await store.initialize();
110
- * store.add("entity1", [0.1, 0.2, 0.3]);
111
- * const results = store.search([0.1, 0.2, 0.3], 5);
112
- * ```
113
- */
114
- export declare class SQLiteVectorStore implements IVectorStore {
115
- /** In-memory cache for fast search */
116
- private memoryStore;
117
- /** SQLite storage reference for persistence */
118
- private storage;
119
- /** Whether the store has been initialized */
120
- private initialized;
121
- /** Model name used for embeddings */
122
- private embeddingModel;
123
- /**
124
- * Create a SQLite vector store.
125
- *
126
- * @param storage - SQLite storage instance with embedding support
127
- * @param embeddingModel - Model name used for embeddings
128
- */
129
- constructor(storage?: SQLiteStorageWithEmbeddings, embeddingModel?: string);
130
- /**
131
- * Initialize the store by loading vectors from SQLite.
132
- */
133
- initialize(): Promise<void>;
134
- /**
135
- * Add a vector for an entity.
136
- * Stores in both memory and SQLite for persistence.
137
- *
138
- * @param entityName - Name of the entity
139
- * @param vector - Embedding vector
140
- */
141
- add(entityName: string, vector: number[]): void;
142
- /**
143
- * Search for similar vectors using cosine similarity.
144
- *
145
- * @param queryVector - Query embedding vector
146
- * @param k - Number of results to return
147
- * @returns Array of results with entity name and similarity score
148
- */
149
- search(queryVector: number[], k: number): VectorSearchResult[];
150
- /**
151
- * Remove a vector by entity name.
152
- *
153
- * @param entityName - Name of the entity to remove
154
- * @returns True if found and removed
155
- */
156
- remove(entityName: string): boolean;
157
- /**
158
- * Get the number of vectors stored.
159
- *
160
- * @returns Number of vectors
161
- */
162
- size(): number;
163
- /**
164
- * Clear all vectors from the store.
165
- */
166
- clear(): void;
167
- /**
168
- * Check if a vector exists for an entity.
169
- *
170
- * @param entityName - Name of the entity
171
- * @returns True if vector exists
172
- */
173
- has(entityName: string): boolean;
174
- /**
175
- * Get the vector for an entity.
176
- *
177
- * @param entityName - Name of the entity
178
- * @returns Vector if found, undefined otherwise
179
- */
180
- get(entityName: string): number[] | undefined;
181
- /**
182
- * Set the SQLite storage reference.
183
- *
184
- * @param storage - SQLite storage instance
185
- */
186
- setStorage(storage: SQLiteStorageWithEmbeddings): void;
187
- /**
188
- * Set the embedding model name.
189
- *
190
- * @param model - Model name
191
- */
192
- setEmbeddingModel(model: string): void;
193
- }
194
- /**
195
- * Interface for SQLite storage with embedding support.
196
- *
197
- * This is a subset of SQLiteStorage that only includes embedding-related methods.
198
- * Allows for loose coupling between VectorStore and SQLiteStorage.
199
- */
200
- export interface SQLiteStorageWithEmbeddings {
201
- /**
202
- * Store an embedding for an entity.
203
- *
204
- * @param entityName - Name of the entity
205
- * @param vector - Embedding vector
206
- * @param model - Model name used for the embedding
207
- */
208
- storeEmbedding(entityName: string, vector: number[], model: string): void;
209
- /**
210
- * Load all embeddings from storage.
211
- *
212
- * @returns Array of [entityName, vector] pairs
213
- */
214
- loadAllEmbeddings(): Promise<[string, number[]][]>;
215
- /**
216
- * Remove an embedding for an entity.
217
- *
218
- * @param entityName - Name of the entity
219
- */
220
- removeEmbedding(entityName: string): void;
221
- /**
222
- * Clear all embeddings from storage.
223
- */
224
- clearAllEmbeddings(): void;
225
- }
226
- /**
227
- * Create a vector store based on storage type.
228
- *
229
- * @param storageType - Storage type: 'jsonl' or 'sqlite'
230
- * @param storage - Optional SQLite storage reference for 'sqlite' type
231
- * @param embeddingModel - Optional model name for embedding tracking
232
- * @returns Vector store instance
233
- */
234
- export declare function createVectorStore(storageType?: 'jsonl' | 'sqlite', storage?: SQLiteStorageWithEmbeddings, embeddingModel?: string): IVectorStore;
235
- //# sourceMappingURL=VectorStore.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"VectorStore.d.ts","sourceRoot":"","sources":["../../src/search/VectorStore.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE1E;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CA2BjE;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,mBAAoB,YAAW,YAAY;IACtD,6CAA6C;IAC7C,OAAO,CAAC,OAAO,CAAoC;IAEnD;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAI/C;;;;;;OAMG;IACH,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,kBAAkB,EAAE;IAuB9D;;;;;OAKG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAInC;;;;OAIG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAIhC;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS;IAI7C;;;;OAIG;IACH,cAAc,IAAI,MAAM,EAAE;IAI1B;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI;CAKtD;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,sCAAsC;IACtC,OAAO,CAAC,WAAW,CAAkD;IAErE,+CAA+C;IAC/C,OAAO,CAAC,OAAO,CAA4C;IAE3D,6CAA6C;IAC7C,OAAO,CAAC,WAAW,CAAS;IAE5B,qCAAqC;IACrC,OAAO,CAAC,cAAc,CAAc;IAEpC;;;;;OAKG;gBACS,OAAO,CAAC,EAAE,2BAA2B,EAAE,cAAc,GAAE,MAAkB;IAKrF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAYjC;;;;;;OAMG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAU/C;;;;;;OAMG;IACH,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,kBAAkB,EAAE;IAI9D;;;;;OAKG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAWnC;;;;OAIG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAIhC;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS;IAI7C;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAItD;;;;OAIG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;CAGvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,2BAA2B;IAC1C;;;;;;OAMG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1E;;;;OAIG;IACH,iBAAiB,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnD;;;;OAIG;IACH,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1C;;OAEG;IACH,kBAAkB,IAAI,IAAI,CAAC;CAC5B;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,WAAW,GAAE,OAAO,GAAG,QAAkB,EACzC,OAAO,CAAC,EAAE,2BAA2B,EACrC,cAAc,CAAC,EAAE,MAAM,GACtB,YAAY,CAQd"}