@danielsimonjr/memoryjs 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/README.md +385 -113
  2. package/README.md.backup-1768084780988 +266 -0
  3. package/dist/index.cjs +17364 -0
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.cts +12371 -0
  6. package/dist/index.d.ts +12367 -11
  7. package/dist/index.js +17102 -19
  8. package/dist/index.js.map +1 -1
  9. package/dist/workers/levenshteinWorker.cjs +102 -0
  10. package/dist/workers/levenshteinWorker.cjs.map +1 -0
  11. package/dist/workers/levenshteinWorker.js +57 -91
  12. package/dist/workers/levenshteinWorker.js.map +1 -1
  13. package/package.json +12 -6
  14. package/dist/core/EntityManager.d.ts +0 -268
  15. package/dist/core/EntityManager.d.ts.map +0 -1
  16. package/dist/core/EntityManager.js +0 -512
  17. package/dist/core/EntityManager.js.map +0 -1
  18. package/dist/core/GraphEventEmitter.d.ts +0 -202
  19. package/dist/core/GraphEventEmitter.d.ts.map +0 -1
  20. package/dist/core/GraphEventEmitter.js +0 -347
  21. package/dist/core/GraphEventEmitter.js.map +0 -1
  22. package/dist/core/GraphStorage.d.ts +0 -395
  23. package/dist/core/GraphStorage.d.ts.map +0 -1
  24. package/dist/core/GraphStorage.js +0 -786
  25. package/dist/core/GraphStorage.js.map +0 -1
  26. package/dist/core/GraphTraversal.d.ts +0 -141
  27. package/dist/core/GraphTraversal.d.ts.map +0 -1
  28. package/dist/core/GraphTraversal.js +0 -574
  29. package/dist/core/GraphTraversal.js.map +0 -1
  30. package/dist/core/HierarchyManager.d.ts +0 -111
  31. package/dist/core/HierarchyManager.d.ts.map +0 -1
  32. package/dist/core/HierarchyManager.js +0 -225
  33. package/dist/core/HierarchyManager.js.map +0 -1
  34. package/dist/core/ManagerContext.d.ts +0 -76
  35. package/dist/core/ManagerContext.d.ts.map +0 -1
  36. package/dist/core/ManagerContext.js +0 -129
  37. package/dist/core/ManagerContext.js.map +0 -1
  38. package/dist/core/ObservationManager.d.ts +0 -85
  39. package/dist/core/ObservationManager.d.ts.map +0 -1
  40. package/dist/core/ObservationManager.js +0 -124
  41. package/dist/core/ObservationManager.js.map +0 -1
  42. package/dist/core/RelationManager.d.ts +0 -131
  43. package/dist/core/RelationManager.d.ts.map +0 -1
  44. package/dist/core/RelationManager.js +0 -212
  45. package/dist/core/RelationManager.js.map +0 -1
  46. package/dist/core/SQLiteStorage.d.ts +0 -354
  47. package/dist/core/SQLiteStorage.d.ts.map +0 -1
  48. package/dist/core/SQLiteStorage.js +0 -919
  49. package/dist/core/SQLiteStorage.js.map +0 -1
  50. package/dist/core/StorageFactory.d.ts +0 -45
  51. package/dist/core/StorageFactory.d.ts.map +0 -1
  52. package/dist/core/StorageFactory.js +0 -65
  53. package/dist/core/StorageFactory.js.map +0 -1
  54. package/dist/core/TransactionManager.d.ts +0 -464
  55. package/dist/core/TransactionManager.d.ts.map +0 -1
  56. package/dist/core/TransactionManager.js +0 -869
  57. package/dist/core/TransactionManager.js.map +0 -1
  58. package/dist/core/index.d.ts +0 -17
  59. package/dist/core/index.d.ts.map +0 -1
  60. package/dist/core/index.js +0 -20
  61. package/dist/core/index.js.map +0 -1
  62. package/dist/features/AnalyticsManager.d.ts +0 -44
  63. package/dist/features/AnalyticsManager.d.ts.map +0 -1
  64. package/dist/features/AnalyticsManager.js +0 -224
  65. package/dist/features/AnalyticsManager.js.map +0 -1
  66. package/dist/features/ArchiveManager.d.ts +0 -133
  67. package/dist/features/ArchiveManager.d.ts.map +0 -1
  68. package/dist/features/ArchiveManager.js +0 -282
  69. package/dist/features/ArchiveManager.js.map +0 -1
  70. package/dist/features/CompressionManager.d.ts +0 -119
  71. package/dist/features/CompressionManager.d.ts.map +0 -1
  72. package/dist/features/CompressionManager.js +0 -470
  73. package/dist/features/CompressionManager.js.map +0 -1
  74. package/dist/features/IOManager.d.ts +0 -225
  75. package/dist/features/IOManager.d.ts.map +0 -1
  76. package/dist/features/IOManager.js +0 -1093
  77. package/dist/features/IOManager.js.map +0 -1
  78. package/dist/features/KeywordExtractor.d.ts +0 -61
  79. package/dist/features/KeywordExtractor.d.ts.map +0 -1
  80. package/dist/features/KeywordExtractor.js +0 -127
  81. package/dist/features/KeywordExtractor.js.map +0 -1
  82. package/dist/features/ObservationNormalizer.d.ts +0 -90
  83. package/dist/features/ObservationNormalizer.d.ts.map +0 -1
  84. package/dist/features/ObservationNormalizer.js +0 -194
  85. package/dist/features/ObservationNormalizer.js.map +0 -1
  86. package/dist/features/StreamingExporter.d.ts +0 -128
  87. package/dist/features/StreamingExporter.d.ts.map +0 -1
  88. package/dist/features/StreamingExporter.js +0 -212
  89. package/dist/features/StreamingExporter.js.map +0 -1
  90. package/dist/features/TagManager.d.ts +0 -147
  91. package/dist/features/TagManager.d.ts.map +0 -1
  92. package/dist/features/TagManager.js +0 -211
  93. package/dist/features/TagManager.js.map +0 -1
  94. package/dist/features/index.d.ts +0 -14
  95. package/dist/features/index.d.ts.map +0 -1
  96. package/dist/features/index.js +0 -15
  97. package/dist/features/index.js.map +0 -1
  98. package/dist/index.d.ts.map +0 -1
  99. package/dist/search/BM25Search.d.ts +0 -148
  100. package/dist/search/BM25Search.d.ts.map +0 -1
  101. package/dist/search/BM25Search.js +0 -340
  102. package/dist/search/BM25Search.js.map +0 -1
  103. package/dist/search/BasicSearch.d.ts +0 -51
  104. package/dist/search/BasicSearch.d.ts.map +0 -1
  105. package/dist/search/BasicSearch.js +0 -138
  106. package/dist/search/BasicSearch.js.map +0 -1
  107. package/dist/search/BooleanSearch.d.ts +0 -98
  108. package/dist/search/BooleanSearch.d.ts.map +0 -1
  109. package/dist/search/BooleanSearch.js +0 -431
  110. package/dist/search/BooleanSearch.js.map +0 -1
  111. package/dist/search/EarlyTerminationManager.d.ts +0 -140
  112. package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
  113. package/dist/search/EarlyTerminationManager.js +0 -280
  114. package/dist/search/EarlyTerminationManager.js.map +0 -1
  115. package/dist/search/EmbeddingCache.d.ts +0 -175
  116. package/dist/search/EmbeddingCache.d.ts.map +0 -1
  117. package/dist/search/EmbeddingCache.js +0 -247
  118. package/dist/search/EmbeddingCache.js.map +0 -1
  119. package/dist/search/EmbeddingService.d.ts +0 -277
  120. package/dist/search/EmbeddingService.d.ts.map +0 -1
  121. package/dist/search/EmbeddingService.js +0 -531
  122. package/dist/search/EmbeddingService.js.map +0 -1
  123. package/dist/search/FuzzySearch.d.ts +0 -118
  124. package/dist/search/FuzzySearch.d.ts.map +0 -1
  125. package/dist/search/FuzzySearch.js +0 -313
  126. package/dist/search/FuzzySearch.js.map +0 -1
  127. package/dist/search/HybridScorer.d.ts +0 -181
  128. package/dist/search/HybridScorer.d.ts.map +0 -1
  129. package/dist/search/HybridScorer.js +0 -258
  130. package/dist/search/HybridScorer.js.map +0 -1
  131. package/dist/search/HybridSearchManager.d.ts +0 -80
  132. package/dist/search/HybridSearchManager.d.ts.map +0 -1
  133. package/dist/search/HybridSearchManager.js +0 -188
  134. package/dist/search/HybridSearchManager.js.map +0 -1
  135. package/dist/search/IncrementalIndexer.d.ts +0 -201
  136. package/dist/search/IncrementalIndexer.d.ts.map +0 -1
  137. package/dist/search/IncrementalIndexer.js +0 -343
  138. package/dist/search/IncrementalIndexer.js.map +0 -1
  139. package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
  140. package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
  141. package/dist/search/OptimizedInvertedIndex.js +0 -359
  142. package/dist/search/OptimizedInvertedIndex.js.map +0 -1
  143. package/dist/search/ParallelSearchExecutor.d.ts +0 -172
  144. package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
  145. package/dist/search/ParallelSearchExecutor.js +0 -310
  146. package/dist/search/ParallelSearchExecutor.js.map +0 -1
  147. package/dist/search/QuantizedVectorStore.d.ts +0 -171
  148. package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
  149. package/dist/search/QuantizedVectorStore.js +0 -308
  150. package/dist/search/QuantizedVectorStore.js.map +0 -1
  151. package/dist/search/QueryAnalyzer.d.ts +0 -76
  152. package/dist/search/QueryAnalyzer.d.ts.map +0 -1
  153. package/dist/search/QueryAnalyzer.js +0 -228
  154. package/dist/search/QueryAnalyzer.js.map +0 -1
  155. package/dist/search/QueryCostEstimator.d.ts +0 -244
  156. package/dist/search/QueryCostEstimator.d.ts.map +0 -1
  157. package/dist/search/QueryCostEstimator.js +0 -653
  158. package/dist/search/QueryCostEstimator.js.map +0 -1
  159. package/dist/search/QueryPlanCache.d.ts +0 -220
  160. package/dist/search/QueryPlanCache.d.ts.map +0 -1
  161. package/dist/search/QueryPlanCache.js +0 -380
  162. package/dist/search/QueryPlanCache.js.map +0 -1
  163. package/dist/search/QueryPlanner.d.ts +0 -58
  164. package/dist/search/QueryPlanner.d.ts.map +0 -1
  165. package/dist/search/QueryPlanner.js +0 -138
  166. package/dist/search/QueryPlanner.js.map +0 -1
  167. package/dist/search/RankedSearch.d.ts +0 -71
  168. package/dist/search/RankedSearch.d.ts.map +0 -1
  169. package/dist/search/RankedSearch.js +0 -239
  170. package/dist/search/RankedSearch.js.map +0 -1
  171. package/dist/search/ReflectionManager.d.ts +0 -120
  172. package/dist/search/ReflectionManager.d.ts.map +0 -1
  173. package/dist/search/ReflectionManager.js +0 -232
  174. package/dist/search/ReflectionManager.js.map +0 -1
  175. package/dist/search/SavedSearchManager.d.ts +0 -79
  176. package/dist/search/SavedSearchManager.d.ts.map +0 -1
  177. package/dist/search/SavedSearchManager.js +0 -147
  178. package/dist/search/SavedSearchManager.js.map +0 -1
  179. package/dist/search/SearchFilterChain.d.ts +0 -120
  180. package/dist/search/SearchFilterChain.d.ts.map +0 -1
  181. package/dist/search/SearchFilterChain.js +0 -186
  182. package/dist/search/SearchFilterChain.js.map +0 -1
  183. package/dist/search/SearchManager.d.ts +0 -326
  184. package/dist/search/SearchManager.d.ts.map +0 -1
  185. package/dist/search/SearchManager.js +0 -454
  186. package/dist/search/SearchManager.js.map +0 -1
  187. package/dist/search/SearchSuggestions.d.ts +0 -27
  188. package/dist/search/SearchSuggestions.d.ts.map +0 -1
  189. package/dist/search/SearchSuggestions.js +0 -58
  190. package/dist/search/SearchSuggestions.js.map +0 -1
  191. package/dist/search/SemanticSearch.d.ts +0 -149
  192. package/dist/search/SemanticSearch.d.ts.map +0 -1
  193. package/dist/search/SemanticSearch.js +0 -324
  194. package/dist/search/SemanticSearch.js.map +0 -1
  195. package/dist/search/SymbolicSearch.d.ts +0 -61
  196. package/dist/search/SymbolicSearch.d.ts.map +0 -1
  197. package/dist/search/SymbolicSearch.js +0 -164
  198. package/dist/search/SymbolicSearch.js.map +0 -1
  199. package/dist/search/TFIDFEventSync.d.ts +0 -85
  200. package/dist/search/TFIDFEventSync.d.ts.map +0 -1
  201. package/dist/search/TFIDFEventSync.js +0 -134
  202. package/dist/search/TFIDFEventSync.js.map +0 -1
  203. package/dist/search/TFIDFIndexManager.d.ts +0 -151
  204. package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
  205. package/dist/search/TFIDFIndexManager.js +0 -433
  206. package/dist/search/TFIDFIndexManager.js.map +0 -1
  207. package/dist/search/VectorStore.d.ts +0 -235
  208. package/dist/search/VectorStore.d.ts.map +0 -1
  209. package/dist/search/VectorStore.js +0 -312
  210. package/dist/search/VectorStore.js.map +0 -1
  211. package/dist/search/index.d.ts +0 -35
  212. package/dist/search/index.d.ts.map +0 -1
  213. package/dist/search/index.js +0 -53
  214. package/dist/search/index.js.map +0 -1
  215. package/dist/types/index.d.ts +0 -13
  216. package/dist/types/index.d.ts.map +0 -1
  217. package/dist/types/index.js +0 -13
  218. package/dist/types/index.js.map +0 -1
  219. package/dist/types/types.d.ts +0 -1811
  220. package/dist/types/types.d.ts.map +0 -1
  221. package/dist/types/types.js +0 -10
  222. package/dist/types/types.js.map +0 -1
  223. package/dist/utils/BatchProcessor.d.ts +0 -271
  224. package/dist/utils/BatchProcessor.d.ts.map +0 -1
  225. package/dist/utils/BatchProcessor.js +0 -377
  226. package/dist/utils/BatchProcessor.js.map +0 -1
  227. package/dist/utils/MemoryMonitor.d.ts +0 -176
  228. package/dist/utils/MemoryMonitor.d.ts.map +0 -1
  229. package/dist/utils/MemoryMonitor.js +0 -306
  230. package/dist/utils/MemoryMonitor.js.map +0 -1
  231. package/dist/utils/WorkerPoolManager.d.ts +0 -233
  232. package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
  233. package/dist/utils/WorkerPoolManager.js +0 -421
  234. package/dist/utils/WorkerPoolManager.js.map +0 -1
  235. package/dist/utils/compressedCache.d.ts +0 -221
  236. package/dist/utils/compressedCache.d.ts.map +0 -1
  237. package/dist/utils/compressedCache.js +0 -349
  238. package/dist/utils/compressedCache.js.map +0 -1
  239. package/dist/utils/compressionUtil.d.ts +0 -214
  240. package/dist/utils/compressionUtil.d.ts.map +0 -1
  241. package/dist/utils/compressionUtil.js +0 -248
  242. package/dist/utils/compressionUtil.js.map +0 -1
  243. package/dist/utils/constants.d.ts +0 -245
  244. package/dist/utils/constants.d.ts.map +0 -1
  245. package/dist/utils/constants.js +0 -253
  246. package/dist/utils/constants.js.map +0 -1
  247. package/dist/utils/entityUtils.d.ts +0 -379
  248. package/dist/utils/entityUtils.d.ts.map +0 -1
  249. package/dist/utils/entityUtils.js +0 -649
  250. package/dist/utils/entityUtils.js.map +0 -1
  251. package/dist/utils/errors.d.ts +0 -95
  252. package/dist/utils/errors.d.ts.map +0 -1
  253. package/dist/utils/errors.js +0 -146
  254. package/dist/utils/errors.js.map +0 -1
  255. package/dist/utils/formatters.d.ts +0 -145
  256. package/dist/utils/formatters.d.ts.map +0 -1
  257. package/dist/utils/formatters.js +0 -133
  258. package/dist/utils/formatters.js.map +0 -1
  259. package/dist/utils/index.d.ts +0 -26
  260. package/dist/utils/index.d.ts.map +0 -1
  261. package/dist/utils/index.js +0 -88
  262. package/dist/utils/index.js.map +0 -1
  263. package/dist/utils/indexes.d.ts +0 -270
  264. package/dist/utils/indexes.d.ts.map +0 -1
  265. package/dist/utils/indexes.js +0 -527
  266. package/dist/utils/indexes.js.map +0 -1
  267. package/dist/utils/logger.d.ts +0 -31
  268. package/dist/utils/logger.d.ts.map +0 -1
  269. package/dist/utils/logger.js +0 -41
  270. package/dist/utils/logger.js.map +0 -1
  271. package/dist/utils/operationUtils.d.ts +0 -124
  272. package/dist/utils/operationUtils.d.ts.map +0 -1
  273. package/dist/utils/operationUtils.js +0 -176
  274. package/dist/utils/operationUtils.js.map +0 -1
  275. package/dist/utils/parallelUtils.d.ts +0 -76
  276. package/dist/utils/parallelUtils.d.ts.map +0 -1
  277. package/dist/utils/parallelUtils.js +0 -192
  278. package/dist/utils/parallelUtils.js.map +0 -1
  279. package/dist/utils/schemas.d.ts +0 -556
  280. package/dist/utils/schemas.d.ts.map +0 -1
  281. package/dist/utils/schemas.js +0 -485
  282. package/dist/utils/schemas.js.map +0 -1
  283. package/dist/utils/searchAlgorithms.d.ts +0 -99
  284. package/dist/utils/searchAlgorithms.d.ts.map +0 -1
  285. package/dist/utils/searchAlgorithms.js +0 -168
  286. package/dist/utils/searchAlgorithms.js.map +0 -1
  287. package/dist/utils/searchCache.d.ts +0 -108
  288. package/dist/utils/searchCache.d.ts.map +0 -1
  289. package/dist/utils/searchCache.js +0 -210
  290. package/dist/utils/searchCache.js.map +0 -1
  291. package/dist/utils/taskScheduler.d.ts +0 -294
  292. package/dist/utils/taskScheduler.d.ts.map +0 -1
  293. package/dist/utils/taskScheduler.js +0 -487
  294. package/dist/utils/taskScheduler.js.map +0 -1
  295. package/dist/workers/index.d.ts +0 -12
  296. package/dist/workers/index.d.ts.map +0 -1
  297. package/dist/workers/index.js +0 -10
  298. package/dist/workers/index.js.map +0 -1
  299. package/dist/workers/levenshteinWorker.d.ts +0 -60
  300. package/dist/workers/levenshteinWorker.d.ts.map +0 -1
@@ -1,786 +0,0 @@
1
- /**
2
- * Graph Storage
3
- *
4
- * Handles file I/O operations for the knowledge graph using JSONL format.
5
- * Implements IGraphStorage interface for storage abstraction.
6
- *
7
- * @module core/GraphStorage
8
- */
9
- import { promises as fs } from 'fs';
10
- import { Mutex } from 'async-mutex';
11
- import { clearAllSearchCaches } from '../utils/searchCache.js';
12
- import { NameIndex, TypeIndex, LowercaseCache, RelationIndex, ObservationIndex } from '../utils/indexes.js';
13
- import { sanitizeObject } from '../utils/index.js';
14
- import { BatchTransaction } from './TransactionManager.js';
15
- import { GraphEventEmitter } from './GraphEventEmitter.js';
16
- /**
17
- * GraphStorage manages persistence of the knowledge graph to disk.
18
- *
19
- * Uses JSONL (JSON Lines) format where each line is a separate JSON object
20
- * representing either an entity or a relation.
21
- *
22
- * OPTIMIZED: Implements in-memory caching to avoid repeated disk reads.
23
- * Cache is invalidated on every write operation to ensure consistency.
24
- *
25
- * @example
26
- * ```typescript
27
- * const storage = new GraphStorage('/path/to/memory.jsonl');
28
- * const graph = await storage.loadGraph();
29
- * graph.entities.push(newEntity);
30
- * await storage.saveGraph(graph);
31
- * ```
32
- */
33
- export class GraphStorage {
34
- memoryFilePath;
35
- /**
36
- * Mutex for thread-safe access to storage operations.
37
- * Prevents concurrent writes from corrupting the file or cache.
38
- */
39
- mutex = new Mutex();
40
- /**
41
- * In-memory cache of the knowledge graph.
42
- * Null when cache is empty or invalidated.
43
- */
44
- cache = null;
45
- /**
46
- * Number of pending append operations since last compaction.
47
- * Used to trigger automatic compaction when threshold is reached.
48
- */
49
- pendingAppends = 0;
50
- /**
51
- * Dynamic threshold for automatic compaction.
52
- *
53
- * Returns the larger of 100 or 10% of the current entity count.
54
- * This scales with graph size to avoid too-frequent compaction on large graphs
55
- * while maintaining a reasonable minimum for small graphs.
56
- *
57
- * @returns Compaction threshold value
58
- */
59
- get compactionThreshold() {
60
- return Math.max(100, Math.floor((this.cache?.entities.length ?? 0) * 0.1));
61
- }
62
- /**
63
- * O(1) entity lookup by name.
64
- */
65
- nameIndex = new NameIndex();
66
- /**
67
- * O(1) entity lookup by type.
68
- */
69
- typeIndex = new TypeIndex();
70
- /**
71
- * Pre-computed lowercase strings for search optimization.
72
- */
73
- lowercaseCache = new LowercaseCache();
74
- /**
75
- * O(1) relation lookup by entity name.
76
- */
77
- relationIndex = new RelationIndex();
78
- /**
79
- * O(1) observation word lookup by entity.
80
- * Maps words in observations to entity names.
81
- */
82
- observationIndex = new ObservationIndex();
83
- /**
84
- * Phase 10 Sprint 2: Event emitter for graph change notifications.
85
- * Allows external systems to subscribe to graph changes.
86
- */
87
- eventEmitter = new GraphEventEmitter();
88
- /**
89
- * Create a new GraphStorage instance.
90
- *
91
- * @param memoryFilePath - Absolute path to the JSONL file
92
- */
93
- constructor(memoryFilePath) {
94
- this.memoryFilePath = memoryFilePath;
95
- }
96
- // ==================== Phase 10 Sprint 2: Event Emitter Access ====================
97
- /**
98
- * Get the event emitter for subscribing to graph changes.
99
- *
100
- * @returns GraphEventEmitter instance
101
- *
102
- * @example
103
- * ```typescript
104
- * const storage = new GraphStorage('/data/memory.jsonl');
105
- *
106
- * // Subscribe to entity creation events
107
- * storage.events.on('entity:created', (event) => {
108
- * console.log(`Entity ${event.entity.name} created`);
109
- * });
110
- *
111
- * // Subscribe to all events
112
- * storage.events.onAny((event) => {
113
- * console.log(`Graph event: ${event.type}`);
114
- * });
115
- * ```
116
- */
117
- get events() {
118
- return this.eventEmitter;
119
- }
120
- // ==================== Durable File Operations ====================
121
- /**
122
- * Write content to file with fsync for durability.
123
- *
124
- * @param content - Content to write
125
- */
126
- async durableWriteFile(content) {
127
- const fd = await fs.open(this.memoryFilePath, 'w');
128
- try {
129
- await fd.write(content);
130
- await fd.sync();
131
- }
132
- finally {
133
- await fd.close();
134
- }
135
- }
136
- /**
137
- * Append content to file with fsync for durability.
138
- *
139
- * @param content - Content to append
140
- * @param prependNewline - Whether to prepend a newline
141
- */
142
- async durableAppendFile(content, prependNewline) {
143
- const fd = await fs.open(this.memoryFilePath, 'a');
144
- try {
145
- const dataToWrite = prependNewline ? '\n' + content : content;
146
- await fd.write(dataToWrite);
147
- await fd.sync();
148
- }
149
- finally {
150
- await fd.close();
151
- }
152
- }
153
- /**
154
- * Load the knowledge graph from disk (read-only access).
155
- *
156
- * OPTIMIZED: Returns cached reference directly without copying.
157
- * This is O(1) regardless of graph size. For mutation operations,
158
- * use getGraphForMutation() instead.
159
- *
160
- * @returns Promise resolving to read-only knowledge graph reference
161
- * @throws Error if file exists but cannot be read or parsed
162
- */
163
- async loadGraph() {
164
- // Return cached graph directly (no copying - O(1))
165
- if (this.cache !== null) {
166
- return this.cache;
167
- }
168
- // Cache miss - load from disk
169
- await this.loadFromDisk();
170
- return this.cache;
171
- }
172
- /**
173
- * Get a mutable copy of the graph for write operations.
174
- *
175
- * Creates deep copies of entity and relation arrays to allow
176
- * safe mutation without affecting the cached data.
177
- *
178
- * @returns Promise resolving to mutable knowledge graph copy
179
- */
180
- async getGraphForMutation() {
181
- await this.ensureLoaded();
182
- return {
183
- entities: this.cache.entities.map(e => ({
184
- ...e,
185
- observations: [...e.observations],
186
- tags: e.tags ? [...e.tags] : undefined,
187
- })),
188
- relations: this.cache.relations.map(r => ({ ...r })),
189
- };
190
- }
191
- /**
192
- * Ensure the cache is loaded from disk.
193
- *
194
- * @returns Promise resolving when cache is populated
195
- */
196
- async ensureLoaded() {
197
- if (this.cache === null) {
198
- await this.loadFromDisk();
199
- }
200
- }
201
- /**
202
- * Internal method to load graph from disk into cache.
203
- */
204
- async loadFromDisk() {
205
- try {
206
- const data = await fs.readFile(this.memoryFilePath, 'utf-8');
207
- const lines = data.split('\n').filter((line) => line.trim() !== '');
208
- // Use Maps to deduplicate - later entries override earlier ones
209
- // This supports append-only updates where new versions are appended
210
- const entityMap = new Map();
211
- const relationMap = new Map();
212
- for (const line of lines) {
213
- const item = JSON.parse(line);
214
- if (item.type === 'entity') {
215
- // Add createdAt if missing for backward compatibility
216
- if (!item.createdAt)
217
- item.createdAt = new Date().toISOString();
218
- // Add lastModified if missing for backward compatibility
219
- if (!item.lastModified)
220
- item.lastModified = item.createdAt;
221
- // Use name as key - later entries override earlier ones
222
- entityMap.set(item.name, item);
223
- }
224
- if (item.type === 'relation') {
225
- // Add createdAt if missing for backward compatibility
226
- if (!item.createdAt)
227
- item.createdAt = new Date().toISOString();
228
- // Add lastModified if missing for backward compatibility
229
- if (!item.lastModified)
230
- item.lastModified = item.createdAt;
231
- // Use composite key for relations
232
- const key = `${item.from}:${item.to}:${item.relationType}`;
233
- relationMap.set(key, item);
234
- }
235
- }
236
- // Convert maps to arrays
237
- const graph = {
238
- entities: Array.from(entityMap.values()),
239
- relations: Array.from(relationMap.values()),
240
- };
241
- // Populate cache
242
- this.cache = graph;
243
- // Build indexes from loaded data
244
- this.buildEntityIndexes(graph.entities);
245
- this.buildRelationIndex(graph.relations);
246
- // Phase 10 Sprint 2: Emit graph:loaded event
247
- this.eventEmitter.emitGraphLoaded(graph.entities.length, graph.relations.length);
248
- }
249
- catch (error) {
250
- // File doesn't exist - create empty graph
251
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
252
- this.cache = { entities: [], relations: [] };
253
- this.clearIndexes();
254
- // Phase 10 Sprint 2: Emit graph:loaded event for empty graph
255
- this.eventEmitter.emitGraphLoaded(0, 0);
256
- return;
257
- }
258
- throw error;
259
- }
260
- }
261
- /**
262
- * Build all entity indexes from entity array.
263
- */
264
- buildEntityIndexes(entities) {
265
- this.nameIndex.build(entities);
266
- this.typeIndex.build(entities);
267
- this.lowercaseCache.build(entities);
268
- // Build observation index
269
- this.observationIndex.clear();
270
- for (const entity of entities) {
271
- this.observationIndex.add(entity.name, entity.observations);
272
- }
273
- }
274
- /**
275
- * Build relation index from relation array.
276
- */
277
- buildRelationIndex(relations) {
278
- this.relationIndex.build(relations);
279
- }
280
- /**
281
- * Clear all indexes.
282
- */
283
- clearIndexes() {
284
- this.nameIndex.clear();
285
- this.typeIndex.clear();
286
- this.lowercaseCache.clear();
287
- this.relationIndex.clear();
288
- this.observationIndex.clear();
289
- }
290
- /**
291
- * Save the knowledge graph to disk.
292
- *
293
- * OPTIMIZED: Updates cache directly after write to avoid re-reading.
294
- * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
295
- *
296
- * Writes the graph to JSONL format, with one JSON object per line.
297
- *
298
- * @param graph - The knowledge graph to save
299
- * @returns Promise resolving when save is complete
300
- * @throws Error if file cannot be written
301
- */
302
- async saveGraph(graph) {
303
- return this.mutex.runExclusive(async () => {
304
- await this.saveGraphInternal(graph);
305
- });
306
- }
307
- /**
308
- * Append a single entity to the file (O(1) write operation).
309
- *
310
- * OPTIMIZED: Uses file append instead of full rewrite.
311
- * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
312
- * Updates cache in-place and triggers compaction when threshold is reached.
313
- *
314
- * @param entity - The entity to append
315
- * @returns Promise resolving when append is complete
316
- */
317
- async appendEntity(entity) {
318
- await this.ensureLoaded();
319
- return this.mutex.runExclusive(async () => {
320
- const entityData = {
321
- type: 'entity',
322
- name: entity.name,
323
- entityType: entity.entityType,
324
- observations: entity.observations,
325
- createdAt: entity.createdAt,
326
- lastModified: entity.lastModified,
327
- };
328
- // Only include optional fields if they exist
329
- if (entity.tags !== undefined)
330
- entityData.tags = entity.tags;
331
- if (entity.importance !== undefined)
332
- entityData.importance = entity.importance;
333
- if (entity.parentId !== undefined)
334
- entityData.parentId = entity.parentId;
335
- const line = JSON.stringify(entityData);
336
- // Append to file with fsync for durability (write FIRST, then update cache)
337
- try {
338
- const stat = await fs.stat(this.memoryFilePath);
339
- await this.durableAppendFile(line, stat.size > 0);
340
- }
341
- catch (error) {
342
- // File doesn't exist - create it
343
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
344
- await this.durableWriteFile(line);
345
- }
346
- else {
347
- throw error;
348
- }
349
- }
350
- // Update cache in-place (after successful file write)
351
- this.cache.entities.push(entity);
352
- // Update indexes
353
- this.nameIndex.add(entity);
354
- this.typeIndex.add(entity);
355
- this.lowercaseCache.set(entity);
356
- this.observationIndex.add(entity.name, entity.observations);
357
- this.pendingAppends++;
358
- // Clear search caches
359
- clearAllSearchCaches();
360
- // Phase 10 Sprint 2: Emit entity:created event
361
- this.eventEmitter.emitEntityCreated(entity);
362
- // Trigger compaction if threshold reached
363
- if (this.pendingAppends >= this.compactionThreshold) {
364
- await this.compactInternal();
365
- }
366
- });
367
- }
368
- /**
369
- * Append a single relation to the file (O(1) write operation).
370
- *
371
- * OPTIMIZED: Uses file append instead of full rewrite.
372
- * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
373
- * Updates cache in-place and triggers compaction when threshold is reached.
374
- *
375
- * @param relation - The relation to append
376
- * @returns Promise resolving when append is complete
377
- */
378
- async appendRelation(relation) {
379
- await this.ensureLoaded();
380
- return this.mutex.runExclusive(async () => {
381
- const line = JSON.stringify({
382
- type: 'relation',
383
- from: relation.from,
384
- to: relation.to,
385
- relationType: relation.relationType,
386
- createdAt: relation.createdAt,
387
- lastModified: relation.lastModified,
388
- });
389
- // Append to file with fsync for durability (write FIRST, then update cache)
390
- try {
391
- const stat = await fs.stat(this.memoryFilePath);
392
- await this.durableAppendFile(line, stat.size > 0);
393
- }
394
- catch (error) {
395
- // File doesn't exist - create it
396
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
397
- await this.durableWriteFile(line);
398
- }
399
- else {
400
- throw error;
401
- }
402
- }
403
- // Update cache in-place (after successful file write)
404
- this.cache.relations.push(relation);
405
- // Update relation index
406
- this.relationIndex.add(relation);
407
- this.pendingAppends++;
408
- // Clear search caches
409
- clearAllSearchCaches();
410
- // Phase 10 Sprint 2: Emit relation:created event
411
- this.eventEmitter.emitRelationCreated(relation);
412
- // Trigger compaction if threshold reached
413
- if (this.pendingAppends >= this.compactionThreshold) {
414
- await this.compactInternal();
415
- }
416
- });
417
- }
418
- /**
419
- * Compact the file by rewriting it with only current cache contents.
420
- *
421
- * THREAD-SAFE: Uses mutex to prevent concurrent operations.
422
- * Removes duplicate entries and cleans up the file.
423
- * Resets pending appends counter.
424
- *
425
- * @returns Promise resolving when compaction is complete
426
- */
427
- async compact() {
428
- return this.mutex.runExclusive(async () => {
429
- await this.compactInternal();
430
- });
431
- }
432
- /**
433
- * Internal compact implementation (must be called within mutex).
434
- *
435
- * @returns Promise resolving when compaction is complete
436
- */
437
- async compactInternal() {
438
- if (this.cache === null) {
439
- return;
440
- }
441
- // Rewrite file with current cache (removes duplicates/updates)
442
- await this.saveGraphInternal(this.cache);
443
- this.pendingAppends = 0;
444
- }
445
- /**
446
- * Internal saveGraph implementation (must be called within mutex).
447
- *
448
- * @param graph - The knowledge graph to save
449
- * @returns Promise resolving when save is complete
450
- */
451
- async saveGraphInternal(graph) {
452
- const lines = [
453
- ...graph.entities.map(e => {
454
- const entityData = {
455
- type: 'entity',
456
- name: e.name,
457
- entityType: e.entityType,
458
- observations: e.observations,
459
- createdAt: e.createdAt,
460
- lastModified: e.lastModified,
461
- };
462
- // Only include optional fields if they exist
463
- if (e.tags !== undefined)
464
- entityData.tags = e.tags;
465
- if (e.importance !== undefined)
466
- entityData.importance = e.importance;
467
- if (e.parentId !== undefined)
468
- entityData.parentId = e.parentId;
469
- return JSON.stringify(entityData);
470
- }),
471
- ...graph.relations.map(r => JSON.stringify({
472
- type: 'relation',
473
- from: r.from,
474
- to: r.to,
475
- relationType: r.relationType,
476
- createdAt: r.createdAt,
477
- lastModified: r.lastModified,
478
- })),
479
- ];
480
- await this.durableWriteFile(lines.join('\n'));
481
- // Update cache directly with the saved graph (avoid re-reading from disk)
482
- this.cache = graph;
483
- // Rebuild indexes with new graph data
484
- this.buildEntityIndexes(graph.entities);
485
- this.buildRelationIndex(graph.relations);
486
- // Reset pending appends since file is now clean
487
- this.pendingAppends = 0;
488
- // Clear all search caches since graph data has changed
489
- clearAllSearchCaches();
490
- // Phase 10 Sprint 2: Emit graph:saved event
491
- this.eventEmitter.emitGraphSaved(graph.entities.length, graph.relations.length);
492
- }
493
- /**
494
- * Get the current pending appends count.
495
- *
496
- * Useful for testing compaction behavior.
497
- *
498
- * @returns Number of pending appends since last compaction
499
- */
500
- getPendingAppends() {
501
- return this.pendingAppends;
502
- }
503
- /**
504
- * Update an entity in-place in the cache and append to file.
505
- *
506
- * OPTIMIZED: Modifies cache directly and appends updated version to file.
507
- * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
508
- * Does not rewrite the entire file - compaction handles deduplication later.
509
- *
510
- * @param entityName - Name of the entity to update
511
- * @param updates - Partial entity updates to apply
512
- * @returns Promise resolving to true if entity was found and updated, false otherwise
513
- */
514
- async updateEntity(entityName, updates) {
515
- await this.ensureLoaded();
516
- return this.mutex.runExclusive(async () => {
517
- const entityIndex = this.cache.entities.findIndex(e => e.name === entityName);
518
- if (entityIndex === -1) {
519
- return false;
520
- }
521
- const entity = this.cache.entities[entityIndex];
522
- const oldType = entity.entityType;
523
- const timestamp = new Date().toISOString();
524
- // Phase 10 Sprint 2: Capture previous values for event
525
- const previousValues = {};
526
- for (const key of Object.keys(updates)) {
527
- if (key in entity) {
528
- previousValues[key] = entity[key];
529
- }
530
- }
531
- // Build the updated entity data for file write BEFORE modifying cache
532
- // This ensures cache consistency if file write fails
533
- const updatedEntity = {
534
- ...entity,
535
- ...updates,
536
- lastModified: timestamp,
537
- };
538
- const entityData = {
539
- type: 'entity',
540
- name: updatedEntity.name,
541
- entityType: updatedEntity.entityType,
542
- observations: updatedEntity.observations,
543
- createdAt: updatedEntity.createdAt,
544
- lastModified: updatedEntity.lastModified,
545
- };
546
- if (updatedEntity.tags !== undefined)
547
- entityData.tags = updatedEntity.tags;
548
- if (updatedEntity.importance !== undefined)
549
- entityData.importance = updatedEntity.importance;
550
- if (updatedEntity.parentId !== undefined)
551
- entityData.parentId = updatedEntity.parentId;
552
- const line = JSON.stringify(entityData);
553
- // Write to file FIRST with durability - if this fails, cache remains consistent
554
- try {
555
- const stat = await fs.stat(this.memoryFilePath);
556
- await this.durableAppendFile(line, stat.size > 0);
557
- }
558
- catch (error) {
559
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
560
- await this.durableWriteFile(line);
561
- }
562
- else {
563
- throw error;
564
- }
565
- }
566
- // File write succeeded - NOW update cache in-place (sanitized to prevent prototype pollution)
567
- Object.assign(entity, sanitizeObject(updates));
568
- entity.lastModified = timestamp;
569
- // Update indexes
570
- this.nameIndex.add(entity); // Update reference
571
- if (updates.entityType && updates.entityType !== oldType) {
572
- this.typeIndex.updateType(entityName, oldType, updates.entityType);
573
- }
574
- this.lowercaseCache.set(entity); // Recompute lowercase
575
- if (updates.observations) {
576
- this.observationIndex.remove(entityName); // Remove old observations
577
- this.observationIndex.add(entityName, entity.observations); // Add new observations
578
- }
579
- this.pendingAppends++;
580
- // Clear search caches
581
- clearAllSearchCaches();
582
- // Phase 10 Sprint 2: Emit entity:updated event
583
- this.eventEmitter.emitEntityUpdated(entityName, updates, previousValues);
584
- // Trigger compaction if threshold reached
585
- if (this.pendingAppends >= this.compactionThreshold) {
586
- await this.compactInternal();
587
- }
588
- return true;
589
- });
590
- }
591
- /**
592
- * Manually clear the cache.
593
- *
594
- * Useful for testing or when external processes modify the file.
595
- *
596
- * @returns void
597
- */
598
- clearCache() {
599
- this.cache = null;
600
- this.clearIndexes();
601
- }
602
- /**
603
- * Get the file path being used for storage.
604
- *
605
- * @returns The memory file path
606
- */
607
- getFilePath() {
608
- return this.memoryFilePath;
609
- }
610
- // ==================== Index Accessors ====================
611
- /**
612
- * Get an entity by name in O(1) time.
613
- *
614
- * OPTIMIZED: Uses NameIndex for constant-time lookup.
615
- *
616
- * @param name - Entity name to look up
617
- * @returns Entity if found, undefined otherwise
618
- */
619
- getEntityByName(name) {
620
- return this.nameIndex.get(name);
621
- }
622
- /**
623
- * Check if an entity exists by name in O(1) time.
624
- *
625
- * @param name - Entity name to check
626
- * @returns True if entity exists
627
- */
628
- hasEntity(name) {
629
- return this.nameIndex.has(name);
630
- }
631
- /**
632
- * Get all entities of a given type in O(1) time.
633
- *
634
- * OPTIMIZED: Uses TypeIndex for constant-time lookup of entity names,
635
- * then uses NameIndex for O(1) entity retrieval.
636
- *
637
- * @param entityType - Entity type to filter by (case-insensitive)
638
- * @returns Array of entities with the given type
639
- */
640
- getEntitiesByType(entityType) {
641
- const names = this.typeIndex.getNames(entityType);
642
- const entities = [];
643
- for (const name of names) {
644
- const entity = this.nameIndex.get(name);
645
- if (entity) {
646
- entities.push(entity);
647
- }
648
- }
649
- return entities;
650
- }
651
- /**
652
- * Get pre-computed lowercase data for an entity.
653
- *
654
- * OPTIMIZED: Avoids repeated toLowerCase() calls during search.
655
- *
656
- * @param entityName - Entity name to get lowercase data for
657
- * @returns LowercaseData if entity exists, undefined otherwise
658
- */
659
- getLowercased(entityName) {
660
- return this.lowercaseCache.get(entityName);
661
- }
662
- /**
663
- * Get all unique entity types in the graph.
664
- *
665
- * @returns Array of unique entity types (lowercase)
666
- */
667
- getEntityTypes() {
668
- return this.typeIndex.getTypes();
669
- }
670
- // ==================== Relation Index Accessors ====================
671
- /**
672
- * Get all relations where the entity is the source (outgoing relations) in O(1) time.
673
- *
674
- * OPTIMIZED: Uses RelationIndex for constant-time lookup.
675
- *
676
- * @param entityName - Entity name to look up outgoing relations for
677
- * @returns Array of relations where entity is the source
678
- */
679
- getRelationsFrom(entityName) {
680
- return this.relationIndex.getRelationsFrom(entityName);
681
- }
682
- /**
683
- * Get all relations where the entity is the target (incoming relations) in O(1) time.
684
- *
685
- * OPTIMIZED: Uses RelationIndex for constant-time lookup.
686
- *
687
- * @param entityName - Entity name to look up incoming relations for
688
- * @returns Array of relations where entity is the target
689
- */
690
- getRelationsTo(entityName) {
691
- return this.relationIndex.getRelationsTo(entityName);
692
- }
693
- /**
694
- * Get all relations involving the entity (both incoming and outgoing) in O(1) time.
695
- *
696
- * OPTIMIZED: Uses RelationIndex for constant-time lookup.
697
- *
698
- * @param entityName - Entity name to look up all relations for
699
- * @returns Array of all relations involving the entity
700
- */
701
- getRelationsFor(entityName) {
702
- return this.relationIndex.getRelationsFor(entityName);
703
- }
704
- /**
705
- * Check if an entity has any relations.
706
- *
707
- * @param entityName - Entity name to check
708
- * @returns True if entity has any relations
709
- */
710
- hasRelations(entityName) {
711
- return this.relationIndex.hasRelations(entityName);
712
- }
713
- // ==================== Observation Index Accessors ====================
714
- /**
715
- * Get entities that have observations containing the given word.
716
- * Uses the observation index for O(1) lookup.
717
- *
718
- * OPTIMIZED: Uses ObservationIndex for constant-time lookup instead of
719
- * linear scan through all entities and their observations.
720
- *
721
- * @param word - Word to search for in observations
722
- * @returns Set of entity names
723
- */
724
- getEntitiesByObservationWord(word) {
725
- return this.observationIndex.getEntitiesWithWord(word);
726
- }
727
- /**
728
- * Get entities that have observations containing ANY of the given words (union).
729
- * Uses the observation index for O(1) lookup per word.
730
- *
731
- * OPTIMIZED: Uses ObservationIndex for constant-time lookups.
732
- *
733
- * @param words - Array of words to search for
734
- * @returns Set of entity names containing any of the words
735
- */
736
- getEntitiesByAnyObservationWord(words) {
737
- return this.observationIndex.getEntitiesWithAnyWord(words);
738
- }
739
- /**
740
- * Get entities that have observations containing ALL of the given words (intersection).
741
- * Uses the observation index for O(1) lookup per word.
742
- *
743
- * OPTIMIZED: Uses ObservationIndex for constant-time lookups and set intersection.
744
- *
745
- * @param words - Array of words that must all be present
746
- * @returns Set of entity names containing all of the words
747
- */
748
- getEntitiesByAllObservationWords(words) {
749
- return this.observationIndex.getEntitiesWithAllWords(words);
750
- }
751
- /**
752
- * Get statistics about the observation index.
753
- *
754
- * @returns Object with wordCount and entityCount
755
- */
756
- getObservationIndexStats() {
757
- return this.observationIndex.getStats();
758
- }
759
- // ==================== Phase 10 Sprint 1: Transaction Factory ====================
760
- /**
761
- * Create a new batch transaction for atomic operations.
762
- *
763
- * Returns a BatchTransaction instance that can be used to queue multiple
764
- * operations and execute them atomically with a single save operation.
765
- *
766
- * @returns A new BatchTransaction instance
767
- *
768
- * @example
769
- * ```typescript
770
- * const storage = new GraphStorage('/data/memory.jsonl');
771
- *
772
- * // Create and execute a batch transaction
773
- * const result = await storage.transaction()
774
- * .createEntity({ name: 'Alice', entityType: 'person', observations: ['Developer'] })
775
- * .createEntity({ name: 'Bob', entityType: 'person', observations: ['Designer'] })
776
- * .createRelation({ from: 'Alice', to: 'Bob', relationType: 'knows' })
777
- * .execute();
778
- *
779
- * console.log(`Batch completed: ${result.operationsExecuted} operations`);
780
- * ```
781
- */
782
- transaction() {
783
- return new BatchTransaction(this);
784
- }
785
- }
786
- //# sourceMappingURL=GraphStorage.js.map