@danielsimonjr/memoryjs 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +266 -0
  3. package/dist/core/EntityManager.d.ts +268 -0
  4. package/dist/core/EntityManager.d.ts.map +1 -0
  5. package/dist/core/EntityManager.js +512 -0
  6. package/dist/core/EntityManager.js.map +1 -0
  7. package/dist/core/GraphEventEmitter.d.ts +202 -0
  8. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  9. package/dist/core/GraphEventEmitter.js +347 -0
  10. package/dist/core/GraphEventEmitter.js.map +1 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +786 -0
  14. package/dist/core/GraphStorage.js.map +1 -0
  15. package/dist/core/GraphTraversal.d.ts +141 -0
  16. package/dist/core/GraphTraversal.d.ts.map +1 -0
  17. package/dist/core/GraphTraversal.js +574 -0
  18. package/dist/core/GraphTraversal.js.map +1 -0
  19. package/dist/core/HierarchyManager.d.ts +111 -0
  20. package/dist/core/HierarchyManager.d.ts.map +1 -0
  21. package/dist/core/HierarchyManager.js +225 -0
  22. package/dist/core/HierarchyManager.js.map +1 -0
  23. package/dist/core/ManagerContext.d.ts +76 -0
  24. package/dist/core/ManagerContext.d.ts.map +1 -0
  25. package/dist/core/ManagerContext.js +129 -0
  26. package/dist/core/ManagerContext.js.map +1 -0
  27. package/dist/core/ObservationManager.d.ts +85 -0
  28. package/dist/core/ObservationManager.d.ts.map +1 -0
  29. package/dist/core/ObservationManager.js +124 -0
  30. package/dist/core/ObservationManager.js.map +1 -0
  31. package/dist/core/RelationManager.d.ts +131 -0
  32. package/dist/core/RelationManager.d.ts.map +1 -0
  33. package/dist/core/RelationManager.js +212 -0
  34. package/dist/core/RelationManager.js.map +1 -0
  35. package/dist/core/SQLiteStorage.d.ts +354 -0
  36. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  37. package/dist/core/SQLiteStorage.js +919 -0
  38. package/dist/core/SQLiteStorage.js.map +1 -0
  39. package/dist/core/StorageFactory.d.ts +45 -0
  40. package/dist/core/StorageFactory.d.ts.map +1 -0
  41. package/dist/core/StorageFactory.js +65 -0
  42. package/dist/core/StorageFactory.js.map +1 -0
  43. package/dist/core/TransactionManager.d.ts +464 -0
  44. package/dist/core/TransactionManager.d.ts.map +1 -0
  45. package/dist/core/TransactionManager.js +869 -0
  46. package/dist/core/TransactionManager.js.map +1 -0
  47. package/dist/core/index.d.ts +17 -0
  48. package/dist/core/index.d.ts.map +1 -0
  49. package/dist/core/index.js +20 -0
  50. package/dist/core/index.js.map +1 -0
  51. package/dist/features/AnalyticsManager.d.ts +44 -0
  52. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  53. package/dist/features/AnalyticsManager.js +224 -0
  54. package/dist/features/AnalyticsManager.js.map +1 -0
  55. package/dist/features/ArchiveManager.d.ts +133 -0
  56. package/dist/features/ArchiveManager.d.ts.map +1 -0
  57. package/dist/features/ArchiveManager.js +282 -0
  58. package/dist/features/ArchiveManager.js.map +1 -0
  59. package/dist/features/CompressionManager.d.ts +119 -0
  60. package/dist/features/CompressionManager.d.ts.map +1 -0
  61. package/dist/features/CompressionManager.js +470 -0
  62. package/dist/features/CompressionManager.js.map +1 -0
  63. package/dist/features/IOManager.d.ts +225 -0
  64. package/dist/features/IOManager.d.ts.map +1 -0
  65. package/dist/features/IOManager.js +1093 -0
  66. package/dist/features/IOManager.js.map +1 -0
  67. package/dist/features/KeywordExtractor.d.ts +61 -0
  68. package/dist/features/KeywordExtractor.d.ts.map +1 -0
  69. package/dist/features/KeywordExtractor.js +127 -0
  70. package/dist/features/KeywordExtractor.js.map +1 -0
  71. package/dist/features/ObservationNormalizer.d.ts +90 -0
  72. package/dist/features/ObservationNormalizer.d.ts.map +1 -0
  73. package/dist/features/ObservationNormalizer.js +194 -0
  74. package/dist/features/ObservationNormalizer.js.map +1 -0
  75. package/dist/features/StreamingExporter.d.ts +128 -0
  76. package/dist/features/StreamingExporter.d.ts.map +1 -0
  77. package/dist/features/StreamingExporter.js +212 -0
  78. package/dist/features/StreamingExporter.js.map +1 -0
  79. package/dist/features/TagManager.d.ts +147 -0
  80. package/dist/features/TagManager.d.ts.map +1 -0
  81. package/dist/features/TagManager.js +211 -0
  82. package/dist/features/TagManager.js.map +1 -0
  83. package/dist/features/index.d.ts +14 -0
  84. package/dist/features/index.d.ts.map +1 -0
  85. package/dist/features/index.js +15 -0
  86. package/dist/features/index.js.map +1 -0
  87. package/dist/index.d.ts +15 -0
  88. package/dist/index.d.ts.map +1 -0
  89. package/dist/index.js +20 -0
  90. package/dist/index.js.map +1 -0
  91. package/dist/search/BM25Search.d.ts +148 -0
  92. package/dist/search/BM25Search.d.ts.map +1 -0
  93. package/dist/search/BM25Search.js +340 -0
  94. package/dist/search/BM25Search.js.map +1 -0
  95. package/dist/search/BasicSearch.d.ts +51 -0
  96. package/dist/search/BasicSearch.d.ts.map +1 -0
  97. package/dist/search/BasicSearch.js +138 -0
  98. package/dist/search/BasicSearch.js.map +1 -0
  99. package/dist/search/BooleanSearch.d.ts +98 -0
  100. package/dist/search/BooleanSearch.d.ts.map +1 -0
  101. package/dist/search/BooleanSearch.js +431 -0
  102. package/dist/search/BooleanSearch.js.map +1 -0
  103. package/dist/search/EarlyTerminationManager.d.ts +140 -0
  104. package/dist/search/EarlyTerminationManager.d.ts.map +1 -0
  105. package/dist/search/EarlyTerminationManager.js +280 -0
  106. package/dist/search/EarlyTerminationManager.js.map +1 -0
  107. package/dist/search/EmbeddingCache.d.ts +175 -0
  108. package/dist/search/EmbeddingCache.d.ts.map +1 -0
  109. package/dist/search/EmbeddingCache.js +247 -0
  110. package/dist/search/EmbeddingCache.js.map +1 -0
  111. package/dist/search/EmbeddingService.d.ts +277 -0
  112. package/dist/search/EmbeddingService.d.ts.map +1 -0
  113. package/dist/search/EmbeddingService.js +531 -0
  114. package/dist/search/EmbeddingService.js.map +1 -0
  115. package/dist/search/FuzzySearch.d.ts +118 -0
  116. package/dist/search/FuzzySearch.d.ts.map +1 -0
  117. package/dist/search/FuzzySearch.js +313 -0
  118. package/dist/search/FuzzySearch.js.map +1 -0
  119. package/dist/search/HybridScorer.d.ts +181 -0
  120. package/dist/search/HybridScorer.d.ts.map +1 -0
  121. package/dist/search/HybridScorer.js +258 -0
  122. package/dist/search/HybridScorer.js.map +1 -0
  123. package/dist/search/HybridSearchManager.d.ts +80 -0
  124. package/dist/search/HybridSearchManager.d.ts.map +1 -0
  125. package/dist/search/HybridSearchManager.js +188 -0
  126. package/dist/search/HybridSearchManager.js.map +1 -0
  127. package/dist/search/IncrementalIndexer.d.ts +201 -0
  128. package/dist/search/IncrementalIndexer.d.ts.map +1 -0
  129. package/dist/search/IncrementalIndexer.js +343 -0
  130. package/dist/search/IncrementalIndexer.js.map +1 -0
  131. package/dist/search/OptimizedInvertedIndex.d.ts +163 -0
  132. package/dist/search/OptimizedInvertedIndex.d.ts.map +1 -0
  133. package/dist/search/OptimizedInvertedIndex.js +359 -0
  134. package/dist/search/OptimizedInvertedIndex.js.map +1 -0
  135. package/dist/search/ParallelSearchExecutor.d.ts +172 -0
  136. package/dist/search/ParallelSearchExecutor.d.ts.map +1 -0
  137. package/dist/search/ParallelSearchExecutor.js +310 -0
  138. package/dist/search/ParallelSearchExecutor.js.map +1 -0
  139. package/dist/search/QuantizedVectorStore.d.ts +171 -0
  140. package/dist/search/QuantizedVectorStore.d.ts.map +1 -0
  141. package/dist/search/QuantizedVectorStore.js +308 -0
  142. package/dist/search/QuantizedVectorStore.js.map +1 -0
  143. package/dist/search/QueryAnalyzer.d.ts +76 -0
  144. package/dist/search/QueryAnalyzer.d.ts.map +1 -0
  145. package/dist/search/QueryAnalyzer.js +228 -0
  146. package/dist/search/QueryAnalyzer.js.map +1 -0
  147. package/dist/search/QueryCostEstimator.d.ts +244 -0
  148. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  149. package/dist/search/QueryCostEstimator.js +653 -0
  150. package/dist/search/QueryCostEstimator.js.map +1 -0
  151. package/dist/search/QueryPlanCache.d.ts +220 -0
  152. package/dist/search/QueryPlanCache.d.ts.map +1 -0
  153. package/dist/search/QueryPlanCache.js +380 -0
  154. package/dist/search/QueryPlanCache.js.map +1 -0
  155. package/dist/search/QueryPlanner.d.ts +58 -0
  156. package/dist/search/QueryPlanner.d.ts.map +1 -0
  157. package/dist/search/QueryPlanner.js +138 -0
  158. package/dist/search/QueryPlanner.js.map +1 -0
  159. package/dist/search/RankedSearch.d.ts +71 -0
  160. package/dist/search/RankedSearch.d.ts.map +1 -0
  161. package/dist/search/RankedSearch.js +239 -0
  162. package/dist/search/RankedSearch.js.map +1 -0
  163. package/dist/search/ReflectionManager.d.ts +120 -0
  164. package/dist/search/ReflectionManager.d.ts.map +1 -0
  165. package/dist/search/ReflectionManager.js +232 -0
  166. package/dist/search/ReflectionManager.js.map +1 -0
  167. package/dist/search/SavedSearchManager.d.ts +79 -0
  168. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  169. package/dist/search/SavedSearchManager.js +147 -0
  170. package/dist/search/SavedSearchManager.js.map +1 -0
  171. package/dist/search/SearchFilterChain.d.ts +120 -0
  172. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  173. package/dist/search/SearchFilterChain.js +186 -0
  174. package/dist/search/SearchFilterChain.js.map +1 -0
  175. package/dist/search/SearchManager.d.ts +326 -0
  176. package/dist/search/SearchManager.d.ts.map +1 -0
  177. package/dist/search/SearchManager.js +454 -0
  178. package/dist/search/SearchManager.js.map +1 -0
  179. package/dist/search/SearchSuggestions.d.ts +27 -0
  180. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  181. package/dist/search/SearchSuggestions.js +58 -0
  182. package/dist/search/SearchSuggestions.js.map +1 -0
  183. package/dist/search/SemanticSearch.d.ts +149 -0
  184. package/dist/search/SemanticSearch.d.ts.map +1 -0
  185. package/dist/search/SemanticSearch.js +324 -0
  186. package/dist/search/SemanticSearch.js.map +1 -0
  187. package/dist/search/SymbolicSearch.d.ts +61 -0
  188. package/dist/search/SymbolicSearch.d.ts.map +1 -0
  189. package/dist/search/SymbolicSearch.js +164 -0
  190. package/dist/search/SymbolicSearch.js.map +1 -0
  191. package/dist/search/TFIDFEventSync.d.ts +85 -0
  192. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  193. package/dist/search/TFIDFEventSync.js +134 -0
  194. package/dist/search/TFIDFEventSync.js.map +1 -0
  195. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  196. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  197. package/dist/search/TFIDFIndexManager.js +433 -0
  198. package/dist/search/TFIDFIndexManager.js.map +1 -0
  199. package/dist/search/VectorStore.d.ts +235 -0
  200. package/dist/search/VectorStore.d.ts.map +1 -0
  201. package/dist/search/VectorStore.js +312 -0
  202. package/dist/search/VectorStore.js.map +1 -0
  203. package/dist/search/index.d.ts +35 -0
  204. package/dist/search/index.d.ts.map +1 -0
  205. package/dist/search/index.js +53 -0
  206. package/dist/search/index.js.map +1 -0
  207. package/dist/types/index.d.ts +13 -0
  208. package/dist/types/index.d.ts.map +1 -0
  209. package/dist/types/index.js +13 -0
  210. package/dist/types/index.js.map +1 -0
  211. package/dist/types/types.d.ts +1811 -0
  212. package/dist/types/types.d.ts.map +1 -0
  213. package/dist/types/types.js +10 -0
  214. package/dist/types/types.js.map +1 -0
  215. package/dist/utils/BatchProcessor.d.ts +271 -0
  216. package/dist/utils/BatchProcessor.d.ts.map +1 -0
  217. package/dist/utils/BatchProcessor.js +377 -0
  218. package/dist/utils/BatchProcessor.js.map +1 -0
  219. package/dist/utils/MemoryMonitor.d.ts +176 -0
  220. package/dist/utils/MemoryMonitor.d.ts.map +1 -0
  221. package/dist/utils/MemoryMonitor.js +306 -0
  222. package/dist/utils/MemoryMonitor.js.map +1 -0
  223. package/dist/utils/WorkerPoolManager.d.ts +233 -0
  224. package/dist/utils/WorkerPoolManager.d.ts.map +1 -0
  225. package/dist/utils/WorkerPoolManager.js +421 -0
  226. package/dist/utils/WorkerPoolManager.js.map +1 -0
  227. package/dist/utils/compressedCache.d.ts +221 -0
  228. package/dist/utils/compressedCache.d.ts.map +1 -0
  229. package/dist/utils/compressedCache.js +349 -0
  230. package/dist/utils/compressedCache.js.map +1 -0
  231. package/dist/utils/compressionUtil.d.ts +214 -0
  232. package/dist/utils/compressionUtil.d.ts.map +1 -0
  233. package/dist/utils/compressionUtil.js +248 -0
  234. package/dist/utils/compressionUtil.js.map +1 -0
  235. package/dist/utils/constants.d.ts +245 -0
  236. package/dist/utils/constants.d.ts.map +1 -0
  237. package/dist/utils/constants.js +253 -0
  238. package/dist/utils/constants.js.map +1 -0
  239. package/dist/utils/entityUtils.d.ts +379 -0
  240. package/dist/utils/entityUtils.d.ts.map +1 -0
  241. package/dist/utils/entityUtils.js +649 -0
  242. package/dist/utils/entityUtils.js.map +1 -0
  243. package/dist/utils/errors.d.ts +95 -0
  244. package/dist/utils/errors.d.ts.map +1 -0
  245. package/dist/utils/errors.js +146 -0
  246. package/dist/utils/errors.js.map +1 -0
  247. package/dist/utils/formatters.d.ts +145 -0
  248. package/dist/utils/formatters.d.ts.map +1 -0
  249. package/dist/utils/formatters.js +133 -0
  250. package/dist/utils/formatters.js.map +1 -0
  251. package/dist/utils/index.d.ts +26 -0
  252. package/dist/utils/index.d.ts.map +1 -0
  253. package/dist/utils/index.js +88 -0
  254. package/dist/utils/index.js.map +1 -0
  255. package/dist/utils/indexes.d.ts +270 -0
  256. package/dist/utils/indexes.d.ts.map +1 -0
  257. package/dist/utils/indexes.js +527 -0
  258. package/dist/utils/indexes.js.map +1 -0
  259. package/dist/utils/logger.d.ts +31 -0
  260. package/dist/utils/logger.d.ts.map +1 -0
  261. package/dist/utils/logger.js +41 -0
  262. package/dist/utils/logger.js.map +1 -0
  263. package/dist/utils/operationUtils.d.ts +124 -0
  264. package/dist/utils/operationUtils.d.ts.map +1 -0
  265. package/dist/utils/operationUtils.js +176 -0
  266. package/dist/utils/operationUtils.js.map +1 -0
  267. package/dist/utils/parallelUtils.d.ts +76 -0
  268. package/dist/utils/parallelUtils.d.ts.map +1 -0
  269. package/dist/utils/parallelUtils.js +192 -0
  270. package/dist/utils/parallelUtils.js.map +1 -0
  271. package/dist/utils/schemas.d.ts +556 -0
  272. package/dist/utils/schemas.d.ts.map +1 -0
  273. package/dist/utils/schemas.js +485 -0
  274. package/dist/utils/schemas.js.map +1 -0
  275. package/dist/utils/searchAlgorithms.d.ts +99 -0
  276. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  277. package/dist/utils/searchAlgorithms.js +168 -0
  278. package/dist/utils/searchAlgorithms.js.map +1 -0
  279. package/dist/utils/searchCache.d.ts +108 -0
  280. package/dist/utils/searchCache.d.ts.map +1 -0
  281. package/dist/utils/searchCache.js +210 -0
  282. package/dist/utils/searchCache.js.map +1 -0
  283. package/dist/utils/taskScheduler.d.ts +294 -0
  284. package/dist/utils/taskScheduler.d.ts.map +1 -0
  285. package/dist/utils/taskScheduler.js +487 -0
  286. package/dist/utils/taskScheduler.js.map +1 -0
  287. package/dist/workers/index.d.ts +12 -0
  288. package/dist/workers/index.d.ts.map +1 -0
  289. package/dist/workers/index.js +10 -0
  290. package/dist/workers/index.js.map +1 -0
  291. package/dist/workers/levenshteinWorker.d.ts +60 -0
  292. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  293. package/dist/workers/levenshteinWorker.js +99 -0
  294. package/dist/workers/levenshteinWorker.js.map +1 -0
  295. package/package.json +69 -0
@@ -0,0 +1,433 @@
1
+ /**
2
+ * TF-IDF Index Manager
3
+ *
4
+ * Manages pre-calculated TF-IDF indexes for fast ranked search.
5
+ * Handles index building, incremental updates, and persistence.
6
+ *
7
+ * @module search/TFIDFIndexManager
8
+ */
9
+ import * as fs from 'fs/promises';
10
+ import * as path from 'path';
11
+ import { calculateIDFFromTokenSets, tokenize } from '../utils/index.js';
12
+ const INDEX_VERSION = '1.0';
13
+ const INDEX_FILENAME = 'tfidf-index.json';
14
+ /**
15
+ * Manages TF-IDF index lifecycle: building, updating, and persistence.
16
+ */
17
+ export class TFIDFIndexManager {
18
+ indexPath;
19
+ index = null;
20
+ constructor(storageDir) {
21
+ this.indexPath = path.join(storageDir, '.indexes', INDEX_FILENAME);
22
+ }
23
+ /**
24
+ * Build a complete TF-IDF index from a knowledge graph.
25
+ *
26
+ * @param graph - Knowledge graph to index
27
+ * @returns Newly built TF-IDF index
28
+ */
29
+ async buildIndex(graph) {
30
+ const documents = new Map();
31
+ const allTokenSets = [];
32
+ // Build document vectors - tokenize once per document
33
+ for (const entity of graph.entities) {
34
+ const documentText = [
35
+ entity.name,
36
+ entity.entityType,
37
+ ...entity.observations,
38
+ ].join(' ');
39
+ const tokens = tokenize(documentText);
40
+ const tokenSet = new Set(tokens);
41
+ allTokenSets.push(tokenSet);
42
+ // Calculate term frequencies
43
+ const termFreq = {};
44
+ for (const term of tokens) {
45
+ termFreq[term] = (termFreq[term] || 0) + 1;
46
+ }
47
+ documents.set(entity.name, {
48
+ entityName: entity.name,
49
+ terms: termFreq,
50
+ documentText,
51
+ });
52
+ }
53
+ // Calculate IDF for all terms using pre-tokenized sets (O(1) lookup per document)
54
+ const idf = new Map();
55
+ const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
56
+ for (const term of allTerms) {
57
+ const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
58
+ idf.set(term, idfScore);
59
+ }
60
+ this.index = {
61
+ version: INDEX_VERSION,
62
+ lastUpdated: new Date().toISOString(),
63
+ documents,
64
+ idf,
65
+ };
66
+ return this.index;
67
+ }
68
+ /**
69
+ * Update the index incrementally when entities change.
70
+ *
71
+ * More efficient than rebuilding the entire index.
72
+ *
73
+ * @param graph - Updated knowledge graph
74
+ * @param changedEntityNames - Names of entities that changed
75
+ */
76
+ async updateIndex(graph, changedEntityNames) {
77
+ if (!this.index) {
78
+ // No existing index, build from scratch
79
+ return this.buildIndex(graph);
80
+ }
81
+ // Rebuild document vectors for changed entities
82
+ const allTokenSets = [];
83
+ const updatedDocuments = new Map(this.index.documents);
84
+ // Remove deleted entities
85
+ for (const entityName of changedEntityNames) {
86
+ const entity = graph.entities.find(e => e.name === entityName);
87
+ if (!entity) {
88
+ updatedDocuments.delete(entityName);
89
+ }
90
+ }
91
+ // Update/add changed entities - tokenize once per document
92
+ for (const entity of graph.entities) {
93
+ const documentText = [
94
+ entity.name,
95
+ entity.entityType,
96
+ ...entity.observations,
97
+ ].join(' ');
98
+ const tokens = tokenize(documentText);
99
+ const tokenSet = new Set(tokens);
100
+ allTokenSets.push(tokenSet);
101
+ if (changedEntityNames.has(entity.name)) {
102
+ // Calculate term frequencies for changed entity
103
+ const termFreq = {};
104
+ for (const term of tokens) {
105
+ termFreq[term] = (termFreq[term] || 0) + 1;
106
+ }
107
+ updatedDocuments.set(entity.name, {
108
+ entityName: entity.name,
109
+ terms: termFreq,
110
+ documentText,
111
+ });
112
+ }
113
+ }
114
+ // Recalculate IDF using pre-tokenized sets (O(1) lookup per document)
115
+ const idf = new Map();
116
+ const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
117
+ for (const term of allTerms) {
118
+ const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
119
+ idf.set(term, idfScore);
120
+ }
121
+ this.index = {
122
+ version: INDEX_VERSION,
123
+ lastUpdated: new Date().toISOString(),
124
+ documents: updatedDocuments,
125
+ idf,
126
+ };
127
+ return this.index;
128
+ }
129
+ /**
130
+ * Load index from disk.
131
+ *
132
+ * @returns Loaded index or null if not found
133
+ */
134
+ async loadIndex() {
135
+ try {
136
+ const data = await fs.readFile(this.indexPath, 'utf-8');
137
+ const serialized = JSON.parse(data);
138
+ this.index = {
139
+ version: serialized.version,
140
+ lastUpdated: serialized.lastUpdated,
141
+ documents: new Map(serialized.documents),
142
+ idf: new Map(serialized.idf),
143
+ };
144
+ return this.index;
145
+ }
146
+ catch (error) {
147
+ // Index doesn't exist or is invalid
148
+ return null;
149
+ }
150
+ }
151
+ /**
152
+ * Save index to disk.
153
+ *
154
+ * @param index - Index to save (uses cached index if not provided)
155
+ */
156
+ async saveIndex(index) {
157
+ const indexToSave = index || this.index;
158
+ if (!indexToSave) {
159
+ throw new Error('No index to save');
160
+ }
161
+ // Ensure index directory exists
162
+ const indexDir = path.dirname(this.indexPath);
163
+ await fs.mkdir(indexDir, { recursive: true });
164
+ // Serialize Map objects to arrays for JSON
165
+ const serialized = {
166
+ version: indexToSave.version,
167
+ lastUpdated: indexToSave.lastUpdated,
168
+ documents: Array.from(indexToSave.documents.entries()),
169
+ idf: Array.from(indexToSave.idf.entries()),
170
+ };
171
+ await fs.writeFile(this.indexPath, JSON.stringify(serialized, null, 2), 'utf-8');
172
+ }
173
+ /**
174
+ * Get the current cached index.
175
+ *
176
+ * @returns Cached index or null if not loaded
177
+ */
178
+ getIndex() {
179
+ return this.index;
180
+ }
181
+ /**
182
+ * Clear the cached index and delete from disk.
183
+ */
184
+ async clearIndex() {
185
+ this.index = null;
186
+ try {
187
+ await fs.unlink(this.indexPath);
188
+ }
189
+ catch {
190
+ // Index file doesn't exist, nothing to delete
191
+ }
192
+ }
193
+ /**
194
+ * Check if the index needs rebuilding based on graph state.
195
+ *
196
+ * @param graph - Current knowledge graph
197
+ * @returns True if index should be rebuilt
198
+ */
199
+ needsRebuild(graph) {
200
+ if (!this.index) {
201
+ return true;
202
+ }
203
+ // Check if entity count matches
204
+ if (this.index.documents.size !== graph.entities.length) {
205
+ return true;
206
+ }
207
+ // Check if all entities are in index
208
+ for (const entity of graph.entities) {
209
+ if (!this.index.documents.has(entity.name)) {
210
+ return true;
211
+ }
212
+ }
213
+ return false;
214
+ }
215
+ // ==================== Phase 10 Sprint 3: Incremental Index Updates ====================
216
+ /**
217
+ * Phase 10 Sprint 3: Add a single document to the index incrementally.
218
+ *
219
+ * More efficient than rebuilding the entire index for single entity additions.
220
+ * Updates TF for the new document and recalculates IDF for affected terms.
221
+ *
222
+ * @param entity - The entity to add
223
+ *
224
+ * @example
225
+ * ```typescript
226
+ * const indexManager = new TFIDFIndexManager('/data');
227
+ * await indexManager.loadIndex();
228
+ *
229
+ * // Add new entity
230
+ * indexManager.addDocument({
231
+ * name: 'NewEntity',
232
+ * entityType: 'person',
233
+ * observations: ['Software engineer']
234
+ * });
235
+ * ```
236
+ */
237
+ addDocument(entity) {
238
+ if (!this.index) {
239
+ // Can't add to non-existent index
240
+ return;
241
+ }
242
+ // Build document text and tokens
243
+ const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
244
+ const tokens = tokenize(documentText);
245
+ // Calculate term frequencies
246
+ const termFreq = {};
247
+ for (const term of tokens) {
248
+ termFreq[term] = (termFreq[term] || 0) + 1;
249
+ }
250
+ // Add to documents map
251
+ this.index.documents.set(entity.name, {
252
+ entityName: entity.name,
253
+ terms: termFreq,
254
+ documentText,
255
+ });
256
+ // Update IDF for ALL terms because N changed (total document count)
257
+ // IDF = log(N/df), and N has increased
258
+ this.recalculateAllIDF();
259
+ // Update timestamp
260
+ this.index.lastUpdated = new Date().toISOString();
261
+ }
262
+ /**
263
+ * Phase 10 Sprint 3: Remove a single document from the index incrementally.
264
+ *
265
+ * More efficient than rebuilding the entire index for single entity deletions.
266
+ * Recalculates IDF for terms that were in the removed document.
267
+ *
268
+ * @param entityName - Name of the entity to remove
269
+ *
270
+ * @example
271
+ * ```typescript
272
+ * indexManager.removeDocument('DeletedEntity');
273
+ * ```
274
+ */
275
+ removeDocument(entityName) {
276
+ if (!this.index) {
277
+ return;
278
+ }
279
+ const document = this.index.documents.get(entityName);
280
+ if (!document) {
281
+ return;
282
+ }
283
+ // Remove from documents map
284
+ this.index.documents.delete(entityName);
285
+ // Update IDF for ALL terms because N changed (total document count)
286
+ // IDF = log(N/df), and N has decreased
287
+ this.recalculateAllIDF();
288
+ // Update timestamp
289
+ this.index.lastUpdated = new Date().toISOString();
290
+ }
291
+ /**
292
+ * Phase 10 Sprint 3: Update a single document in the index incrementally.
293
+ *
294
+ * More efficient than rebuilding the entire index for single entity updates.
295
+ * Handles both term changes and observation updates.
296
+ *
297
+ * @param entity - The updated entity
298
+ *
299
+ * @example
300
+ * ```typescript
301
+ * indexManager.updateDocument({
302
+ * name: 'ExistingEntity',
303
+ * entityType: 'person',
304
+ * observations: ['Updated observations']
305
+ * });
306
+ * ```
307
+ */
308
+ updateDocument(entity) {
309
+ if (!this.index) {
310
+ return;
311
+ }
312
+ const oldDocument = this.index.documents.get(entity.name);
313
+ const oldTerms = oldDocument ? new Set(Object.keys(oldDocument.terms)) : new Set();
314
+ // Build new document
315
+ const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
316
+ const tokens = tokenize(documentText);
317
+ const newTerms = new Set(tokens);
318
+ // Calculate term frequencies
319
+ const termFreq = {};
320
+ for (const term of tokens) {
321
+ termFreq[term] = (termFreq[term] || 0) + 1;
322
+ }
323
+ // Update documents map
324
+ this.index.documents.set(entity.name, {
325
+ entityName: entity.name,
326
+ terms: termFreq,
327
+ documentText,
328
+ });
329
+ // Find terms that changed (added or removed)
330
+ const changedTerms = new Set();
331
+ for (const term of oldTerms) {
332
+ if (!newTerms.has(term)) {
333
+ changedTerms.add(term);
334
+ }
335
+ }
336
+ for (const term of newTerms) {
337
+ if (!oldTerms.has(term)) {
338
+ changedTerms.add(term);
339
+ }
340
+ }
341
+ // Recalculate IDF for changed terms
342
+ if (changedTerms.size > 0) {
343
+ this.recalculateIDFForTerms(changedTerms);
344
+ }
345
+ // Update timestamp
346
+ this.index.lastUpdated = new Date().toISOString();
347
+ }
348
+ /**
349
+ * Phase 10 Sprint 3: Recalculate IDF scores for a set of terms.
350
+ *
351
+ * @param terms - Set of terms to recalculate IDF for
352
+ * @private
353
+ */
354
+ recalculateIDFForTerms(terms) {
355
+ if (!this.index) {
356
+ return;
357
+ }
358
+ const totalDocs = this.index.documents.size;
359
+ if (totalDocs === 0) {
360
+ // No documents, clear all IDF for these terms
361
+ for (const term of terms) {
362
+ this.index.idf.delete(term);
363
+ }
364
+ return;
365
+ }
366
+ // Count documents containing each term
367
+ for (const term of terms) {
368
+ let docCount = 0;
369
+ for (const doc of this.index.documents.values()) {
370
+ if (term in doc.terms) {
371
+ docCount++;
372
+ }
373
+ }
374
+ if (docCount > 0) {
375
+ // IDF = log(N / df) where N = total docs, df = doc frequency
376
+ const idfScore = Math.log(totalDocs / docCount);
377
+ this.index.idf.set(term, idfScore);
378
+ }
379
+ else {
380
+ // Term no longer exists in any document
381
+ this.index.idf.delete(term);
382
+ }
383
+ }
384
+ }
385
+ /**
386
+ * Phase 10 Sprint 3: Recalculate IDF scores for ALL terms in the index.
387
+ *
388
+ * Called when the total document count changes (add/remove document).
389
+ * @private
390
+ */
391
+ recalculateAllIDF() {
392
+ if (!this.index) {
393
+ return;
394
+ }
395
+ const totalDocs = this.index.documents.size;
396
+ if (totalDocs === 0) {
397
+ // No documents, clear all IDF
398
+ this.index.idf.clear();
399
+ return;
400
+ }
401
+ // Build term -> document count map
402
+ const termDocCounts = new Map();
403
+ for (const doc of this.index.documents.values()) {
404
+ for (const term of Object.keys(doc.terms)) {
405
+ termDocCounts.set(term, (termDocCounts.get(term) ?? 0) + 1);
406
+ }
407
+ }
408
+ // Clear old IDF and recalculate
409
+ this.index.idf.clear();
410
+ for (const [term, docCount] of termDocCounts) {
411
+ // IDF = log(N / df) where N = total docs, df = doc frequency
412
+ const idfScore = Math.log(totalDocs / docCount);
413
+ this.index.idf.set(term, idfScore);
414
+ }
415
+ }
416
+ /**
417
+ * Phase 10 Sprint 3: Check if the index is loaded/initialized.
418
+ *
419
+ * @returns True if index is available
420
+ */
421
+ isInitialized() {
422
+ return this.index !== null;
423
+ }
424
+ /**
425
+ * Phase 10 Sprint 3: Get the number of documents in the index.
426
+ *
427
+ * @returns Document count or 0 if not initialized
428
+ */
429
+ getDocumentCount() {
430
+ return this.index?.documents.size ?? 0;
431
+ }
432
+ }
433
+ //# sourceMappingURL=TFIDFIndexManager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"TFIDFIndexManager.js","sourceRoot":"","sources":["../../src/search/TFIDFIndexManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAExE,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAY1C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,SAAS,CAAS;IAClB,KAAK,GAAsB,IAAI,CAAC;IAExC,YAAY,UAAkB;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;QACpD,MAAM,YAAY,GAAkB,EAAE,CAAC;QAEvC,sDAAsD;QACtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG;gBACnB,MAAM,CAAC,IAAI;gBACX,MAAM,CAAC,UAAU;gBACjB,GAAG,MAAM,CAAC,YAAY;aACvB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE5B,6BAA6B;YAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7C,CAAC;YAED,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;gBACzB,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,KAAK,EAAE,QAAQ;gBACf,YAAY;aACb,CAAC,CAAC;QACL,CAAC;QAED,kFAAkF;QAClF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAC/D,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,KAAK,GAAG;YACX,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS;YACT,GAAG;SACJ,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,KAA6B,EAAE,kBAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,wCAAwC;YACxC,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,gDAAgD;QAChD,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEvD,0BAA0B;QAC1B,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;YAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG;gBACnB,MAAM,CAAC,IAAI;gBACX,MAAM,CAAC,UAAU;gBACjB,GAAG,MAAM,CAAC,YAAY;aACvB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE5B,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,gDAAgD;gBAChD,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC7C,CAAC;gBAED,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;oBAChC,UAAU,EAAE,MAAM,CAAC,IAAI;oBACvB,KAAK,EAAE,QAAQ;oBACf,YAAY;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAC/D,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,KAAK,GAAG;YACX,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,gBAAgB;YAC3B,GAAG;SACJ,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,UAAU,GAAyB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE1D,IAAI,CAAC,KAAK,GAAG;gBACX,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,SAAS,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,GAAG,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;aAC7B,CAAC;YAEF,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,oCAAoC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS,CAAC,KAAkB;QAChC,MAAM,WAAW,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACxC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9C,2CAA2C;QAC3C,MAAM,UAAU,GAAyB;YACvC,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACtD,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;SAC3C,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACnF,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,8CAA8C;QAChD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,KAAqB;QAChC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qCAAqC;QACrC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3C,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yFAAyF;IAEzF;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,WAAW,CAAC,MAAoE;QAC9E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,kCAAkC;YAClC,OAAO;QACT,CAAC;QAED,iCAAiC;QACjC,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEtC,6BAA6B;QAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ;YACf,YAAY;SACb,CAAC,CAAC;QAEH,oEAAoE;QACpE,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,cAAc,CAAC,UAAkB;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAExC,oEAAoE;QACpE,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,MAAoE;QACjF,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAE3F,qBAAqB;QACrB,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAEjC,6BAA6B;QAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ;YACf,YAAY;SACb,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACK,sBAAsB,CAAC,KAAkB;QAC/C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,8CAA8C;YAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;gBAChD,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACtB,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,6DAA6D;gBAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,iBAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;QAE5C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,8BAA8B;YAC9B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YAC7C,6DAA6D;YAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC;IACzC,CAAC;CACF"}
@@ -0,0 +1,235 @@
1
+ /**
2
+ * Vector Store
3
+ *
4
+ * Phase 4 Sprint 11: Vector storage and retrieval for semantic search.
5
+ * Provides in-memory and SQLite-backed implementations.
6
+ *
7
+ * @module search/VectorStore
8
+ */
9
+ import type { IVectorStore, VectorSearchResult } from '../types/index.js';
10
+ /**
11
+ * Calculate cosine similarity between two vectors.
12
+ *
13
+ * Uses an optimized inner loop without array methods for maximum performance.
14
+ *
15
+ * @param a - First vector
16
+ * @param b - Second vector
17
+ * @returns Cosine similarity score (0.0 to 1.0)
18
+ */
19
+ export declare function cosineSimilarity(a: number[], b: number[]): number;
20
+ /**
21
+ * In-Memory Vector Store
22
+ *
23
+ * Stores vectors in memory using a Map for O(1) add/remove operations.
24
+ * Search uses brute-force cosine similarity which is O(n) but fast for
25
+ * small to medium graphs (<10K entities).
26
+ *
27
+ * @example
28
+ * ```typescript
29
+ * const store = new InMemoryVectorStore();
30
+ * store.add("entity1", [0.1, 0.2, 0.3]);
31
+ * store.add("entity2", [0.4, 0.5, 0.6]);
32
+ * const results = store.search([0.1, 0.2, 0.3], 5);
33
+ * console.log(results); // [{ name: "entity1", score: 1.0 }, ...]
34
+ * ```
35
+ */
36
+ export declare class InMemoryVectorStore implements IVectorStore {
37
+ /** Map of entity name to embedding vector */
38
+ private vectors;
39
+ /**
40
+ * Add a vector for an entity.
41
+ *
42
+ * @param entityName - Name of the entity
43
+ * @param vector - Embedding vector
44
+ */
45
+ add(entityName: string, vector: number[]): void;
46
+ /**
47
+ * Search for similar vectors using cosine similarity.
48
+ *
49
+ * @param queryVector - Query embedding vector
50
+ * @param k - Number of results to return
51
+ * @returns Array of results with entity name and similarity score
52
+ */
53
+ search(queryVector: number[], k: number): VectorSearchResult[];
54
+ /**
55
+ * Remove a vector by entity name.
56
+ *
57
+ * @param entityName - Name of the entity to remove
58
+ * @returns True if found and removed
59
+ */
60
+ remove(entityName: string): boolean;
61
+ /**
62
+ * Get the number of vectors stored.
63
+ *
64
+ * @returns Number of vectors
65
+ */
66
+ size(): number;
67
+ /**
68
+ * Clear all vectors from the store.
69
+ */
70
+ clear(): void;
71
+ /**
72
+ * Check if a vector exists for an entity.
73
+ *
74
+ * @param entityName - Name of the entity
75
+ * @returns True if vector exists
76
+ */
77
+ has(entityName: string): boolean;
78
+ /**
79
+ * Get the vector for an entity.
80
+ *
81
+ * @param entityName - Name of the entity
82
+ * @returns Vector if found, undefined otherwise
83
+ */
84
+ get(entityName: string): number[] | undefined;
85
+ /**
86
+ * Get all entity names with stored vectors.
87
+ *
88
+ * @returns Array of entity names
89
+ */
90
+ getEntityNames(): string[];
91
+ /**
92
+ * Load vectors from an iterable source.
93
+ *
94
+ * @param entries - Iterable of [entityName, vector] pairs
95
+ */
96
+ loadFrom(entries: Iterable<[string, number[]]>): void;
97
+ }
98
+ /**
99
+ * SQLite Vector Store
100
+ *
101
+ * Persists vectors to SQLite storage while maintaining an in-memory cache
102
+ * for fast search operations. Combines persistence with performance.
103
+ *
104
+ * Uses SQLiteStorage's embedding storage methods for persistence.
105
+ *
106
+ * @example
107
+ * ```typescript
108
+ * const store = new SQLiteVectorStore(sqliteStorage);
109
+ * await store.initialize();
110
+ * store.add("entity1", [0.1, 0.2, 0.3]);
111
+ * const results = store.search([0.1, 0.2, 0.3], 5);
112
+ * ```
113
+ */
114
+ export declare class SQLiteVectorStore implements IVectorStore {
115
+ /** In-memory cache for fast search */
116
+ private memoryStore;
117
+ /** SQLite storage reference for persistence */
118
+ private storage;
119
+ /** Whether the store has been initialized */
120
+ private initialized;
121
+ /** Model name used for embeddings */
122
+ private embeddingModel;
123
+ /**
124
+ * Create a SQLite vector store.
125
+ *
126
+ * @param storage - SQLite storage instance with embedding support
127
+ * @param embeddingModel - Model name used for embeddings
128
+ */
129
+ constructor(storage?: SQLiteStorageWithEmbeddings, embeddingModel?: string);
130
+ /**
131
+ * Initialize the store by loading vectors from SQLite.
132
+ */
133
+ initialize(): Promise<void>;
134
+ /**
135
+ * Add a vector for an entity.
136
+ * Stores in both memory and SQLite for persistence.
137
+ *
138
+ * @param entityName - Name of the entity
139
+ * @param vector - Embedding vector
140
+ */
141
+ add(entityName: string, vector: number[]): void;
142
+ /**
143
+ * Search for similar vectors using cosine similarity.
144
+ *
145
+ * @param queryVector - Query embedding vector
146
+ * @param k - Number of results to return
147
+ * @returns Array of results with entity name and similarity score
148
+ */
149
+ search(queryVector: number[], k: number): VectorSearchResult[];
150
+ /**
151
+ * Remove a vector by entity name.
152
+ *
153
+ * @param entityName - Name of the entity to remove
154
+ * @returns True if found and removed
155
+ */
156
+ remove(entityName: string): boolean;
157
+ /**
158
+ * Get the number of vectors stored.
159
+ *
160
+ * @returns Number of vectors
161
+ */
162
+ size(): number;
163
+ /**
164
+ * Clear all vectors from the store.
165
+ */
166
+ clear(): void;
167
+ /**
168
+ * Check if a vector exists for an entity.
169
+ *
170
+ * @param entityName - Name of the entity
171
+ * @returns True if vector exists
172
+ */
173
+ has(entityName: string): boolean;
174
+ /**
175
+ * Get the vector for an entity.
176
+ *
177
+ * @param entityName - Name of the entity
178
+ * @returns Vector if found, undefined otherwise
179
+ */
180
+ get(entityName: string): number[] | undefined;
181
+ /**
182
+ * Set the SQLite storage reference.
183
+ *
184
+ * @param storage - SQLite storage instance
185
+ */
186
+ setStorage(storage: SQLiteStorageWithEmbeddings): void;
187
+ /**
188
+ * Set the embedding model name.
189
+ *
190
+ * @param model - Model name
191
+ */
192
+ setEmbeddingModel(model: string): void;
193
+ }
194
+ /**
195
+ * Interface for SQLite storage with embedding support.
196
+ *
197
+ * This is a subset of SQLiteStorage that only includes embedding-related methods.
198
+ * Allows for loose coupling between VectorStore and SQLiteStorage.
199
+ */
200
+ export interface SQLiteStorageWithEmbeddings {
201
+ /**
202
+ * Store an embedding for an entity.
203
+ *
204
+ * @param entityName - Name of the entity
205
+ * @param vector - Embedding vector
206
+ * @param model - Model name used for the embedding
207
+ */
208
+ storeEmbedding(entityName: string, vector: number[], model: string): void;
209
+ /**
210
+ * Load all embeddings from storage.
211
+ *
212
+ * @returns Array of [entityName, vector] pairs
213
+ */
214
+ loadAllEmbeddings(): Promise<[string, number[]][]>;
215
+ /**
216
+ * Remove an embedding for an entity.
217
+ *
218
+ * @param entityName - Name of the entity
219
+ */
220
+ removeEmbedding(entityName: string): void;
221
+ /**
222
+ * Clear all embeddings from storage.
223
+ */
224
+ clearAllEmbeddings(): void;
225
+ }
226
+ /**
227
+ * Create a vector store based on storage type.
228
+ *
229
+ * @param storageType - Storage type: 'jsonl' or 'sqlite'
230
+ * @param storage - Optional SQLite storage reference for 'sqlite' type
231
+ * @param embeddingModel - Optional model name for embedding tracking
232
+ * @returns Vector store instance
233
+ */
234
+ export declare function createVectorStore(storageType?: 'jsonl' | 'sqlite', storage?: SQLiteStorageWithEmbeddings, embeddingModel?: string): IVectorStore;
235
+ //# sourceMappingURL=VectorStore.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VectorStore.d.ts","sourceRoot":"","sources":["../../src/search/VectorStore.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE1E;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CA2BjE;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,mBAAoB,YAAW,YAAY;IACtD,6CAA6C;IAC7C,OAAO,CAAC,OAAO,CAAoC;IAEnD;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAI/C;;;;;;OAMG;IACH,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,kBAAkB,EAAE;IAuB9D;;;;;OAKG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAInC;;;;OAIG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAIhC;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS;IAI7C;;;;OAIG;IACH,cAAc,IAAI,MAAM,EAAE;IAI1B;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI;CAKtD;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,sCAAsC;IACtC,OAAO,CAAC,WAAW,CAAkD;IAErE,+CAA+C;IAC/C,OAAO,CAAC,OAAO,CAA4C;IAE3D,6CAA6C;IAC7C,OAAO,CAAC,WAAW,CAAS;IAE5B,qCAAqC;IACrC,OAAO,CAAC,cAAc,CAAc;IAEpC;;;;;OAKG;gBACS,OAAO,CAAC,EAAE,2BAA2B,EAAE,cAAc,GAAE,MAAkB;IAKrF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAYjC;;;;;;OAMG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAU/C;;;;;;OAMG;IACH,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,kBAAkB,EAAE;IAI9D;;;;;OAKG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAWnC;;;;OAIG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAIhC;;;;;OAKG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS;IAI7C;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAItD;;;;OAIG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;CAGvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,2BAA2B;IAC1C;;;;;;OAMG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1E;;;;OAIG;IACH,iBAAiB,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnD;;;;OAIG;IACH,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1C;;OAEG;IACH,kBAAkB,IAAI,IAAI,CAAC;CAC5B;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,WAAW,GAAE,OAAO,GAAG,QAAkB,EACzC,OAAO,CAAC,EAAE,2BAA2B,EACrC,cAAc,CAAC,EAAE,MAAM,GACtB,YAAY,CAQd"}