@danielsimonjr/memory-mcp 0.47.1 → 9.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +2000 -194
  3. package/dist/__tests__/file-path.test.js +5 -5
  4. package/dist/__tests__/knowledge-graph.test.js +3 -8
  5. package/dist/core/EntityManager.d.ts +266 -0
  6. package/dist/core/EntityManager.d.ts.map +1 -0
  7. package/dist/core/EntityManager.js +85 -133
  8. package/dist/core/GraphEventEmitter.d.ts +202 -0
  9. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  10. package/dist/core/GraphEventEmitter.js +346 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +643 -31
  14. package/dist/core/GraphTraversal.d.ts +141 -0
  15. package/dist/core/GraphTraversal.d.ts.map +1 -0
  16. package/dist/core/GraphTraversal.js +573 -0
  17. package/dist/core/HierarchyManager.d.ts +111 -0
  18. package/dist/core/HierarchyManager.d.ts.map +1 -0
  19. package/dist/{features → core}/HierarchyManager.js +14 -9
  20. package/dist/core/ManagerContext.d.ts +72 -0
  21. package/dist/core/ManagerContext.d.ts.map +1 -0
  22. package/dist/core/ManagerContext.js +118 -0
  23. package/dist/core/ObservationManager.d.ts +85 -0
  24. package/dist/core/ObservationManager.d.ts.map +1 -0
  25. package/dist/core/ObservationManager.js +51 -57
  26. package/dist/core/RelationManager.d.ts +131 -0
  27. package/dist/core/RelationManager.d.ts.map +1 -0
  28. package/dist/core/RelationManager.js +31 -7
  29. package/dist/core/SQLiteStorage.d.ts +354 -0
  30. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  31. package/dist/core/SQLiteStorage.js +917 -0
  32. package/dist/core/StorageFactory.d.ts +45 -0
  33. package/dist/core/StorageFactory.d.ts.map +1 -0
  34. package/dist/core/StorageFactory.js +64 -0
  35. package/dist/core/TransactionManager.d.ts +464 -0
  36. package/dist/core/TransactionManager.d.ts.map +1 -0
  37. package/dist/core/TransactionManager.js +490 -13
  38. package/dist/core/index.d.ts +17 -0
  39. package/dist/core/index.d.ts.map +1 -0
  40. package/dist/core/index.js +12 -2
  41. package/dist/features/AnalyticsManager.d.ts +44 -0
  42. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  43. package/dist/features/AnalyticsManager.js +14 -13
  44. package/dist/features/ArchiveManager.d.ts +133 -0
  45. package/dist/features/ArchiveManager.d.ts.map +1 -0
  46. package/dist/features/ArchiveManager.js +221 -14
  47. package/dist/features/CompressionManager.d.ts +117 -0
  48. package/dist/features/CompressionManager.d.ts.map +1 -0
  49. package/dist/features/CompressionManager.js +189 -20
  50. package/dist/features/IOManager.d.ts +225 -0
  51. package/dist/features/IOManager.d.ts.map +1 -0
  52. package/dist/features/IOManager.js +1041 -0
  53. package/dist/features/StreamingExporter.d.ts +123 -0
  54. package/dist/features/StreamingExporter.d.ts.map +1 -0
  55. package/dist/features/StreamingExporter.js +203 -0
  56. package/dist/features/TagManager.d.ts +147 -0
  57. package/dist/features/TagManager.d.ts.map +1 -0
  58. package/dist/features/index.d.ts +12 -0
  59. package/dist/features/index.d.ts.map +1 -0
  60. package/dist/features/index.js +5 -6
  61. package/dist/index.d.ts +9 -0
  62. package/dist/index.d.ts.map +1 -0
  63. package/dist/index.js +12 -45
  64. package/dist/memory.jsonl +1 -18
  65. package/dist/search/BasicSearch.d.ts +51 -0
  66. package/dist/search/BasicSearch.d.ts.map +1 -0
  67. package/dist/search/BasicSearch.js +9 -3
  68. package/dist/search/BooleanSearch.d.ts +98 -0
  69. package/dist/search/BooleanSearch.d.ts.map +1 -0
  70. package/dist/search/BooleanSearch.js +156 -9
  71. package/dist/search/EmbeddingService.d.ts +178 -0
  72. package/dist/search/EmbeddingService.d.ts.map +1 -0
  73. package/dist/search/EmbeddingService.js +358 -0
  74. package/dist/search/FuzzySearch.d.ts +118 -0
  75. package/dist/search/FuzzySearch.d.ts.map +1 -0
  76. package/dist/search/FuzzySearch.js +241 -25
  77. package/dist/search/QueryCostEstimator.d.ts +111 -0
  78. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  79. package/dist/search/QueryCostEstimator.js +355 -0
  80. package/dist/search/RankedSearch.d.ts +71 -0
  81. package/dist/search/RankedSearch.d.ts.map +1 -0
  82. package/dist/search/RankedSearch.js +54 -6
  83. package/dist/search/SavedSearchManager.d.ts +79 -0
  84. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  85. package/dist/search/SearchFilterChain.d.ts +120 -0
  86. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  87. package/dist/search/SearchFilterChain.js +2 -4
  88. package/dist/search/SearchManager.d.ts +326 -0
  89. package/dist/search/SearchManager.d.ts.map +1 -0
  90. package/dist/search/SearchManager.js +148 -0
  91. package/dist/search/SearchSuggestions.d.ts +27 -0
  92. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  93. package/dist/search/SearchSuggestions.js +1 -1
  94. package/dist/search/SemanticSearch.d.ts +149 -0
  95. package/dist/search/SemanticSearch.d.ts.map +1 -0
  96. package/dist/search/SemanticSearch.js +323 -0
  97. package/dist/search/TFIDFEventSync.d.ts +85 -0
  98. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  99. package/dist/search/TFIDFEventSync.js +133 -0
  100. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  101. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  102. package/dist/search/TFIDFIndexManager.js +232 -17
  103. package/dist/search/VectorStore.d.ts +235 -0
  104. package/dist/search/VectorStore.d.ts.map +1 -0
  105. package/dist/search/VectorStore.js +311 -0
  106. package/dist/search/index.d.ts +21 -0
  107. package/dist/search/index.d.ts.map +1 -0
  108. package/dist/search/index.js +12 -0
  109. package/dist/server/MCPServer.d.ts +21 -0
  110. package/dist/server/MCPServer.d.ts.map +1 -0
  111. package/dist/server/MCPServer.js +4 -4
  112. package/dist/server/responseCompressor.d.ts +94 -0
  113. package/dist/server/responseCompressor.d.ts.map +1 -0
  114. package/dist/server/responseCompressor.js +127 -0
  115. package/dist/server/toolDefinitions.d.ts +27 -0
  116. package/dist/server/toolDefinitions.d.ts.map +1 -0
  117. package/dist/server/toolDefinitions.js +189 -18
  118. package/dist/server/toolHandlers.d.ts +41 -0
  119. package/dist/server/toolHandlers.d.ts.map +1 -0
  120. package/dist/server/toolHandlers.js +467 -75
  121. package/dist/types/index.d.ts +13 -0
  122. package/dist/types/index.d.ts.map +1 -0
  123. package/dist/types/index.js +1 -1
  124. package/dist/types/types.d.ts +1654 -0
  125. package/dist/types/types.d.ts.map +1 -0
  126. package/dist/types/types.js +9 -0
  127. package/dist/utils/compressedCache.d.ts +192 -0
  128. package/dist/utils/compressedCache.d.ts.map +1 -0
  129. package/dist/utils/compressedCache.js +309 -0
  130. package/dist/utils/compressionUtil.d.ts +214 -0
  131. package/dist/utils/compressionUtil.d.ts.map +1 -0
  132. package/dist/utils/compressionUtil.js +247 -0
  133. package/dist/utils/constants.d.ts +245 -0
  134. package/dist/utils/constants.d.ts.map +1 -0
  135. package/dist/utils/constants.js +124 -0
  136. package/dist/utils/entityUtils.d.ts +321 -0
  137. package/dist/utils/entityUtils.d.ts.map +1 -0
  138. package/dist/utils/entityUtils.js +434 -4
  139. package/dist/utils/errors.d.ts +95 -0
  140. package/dist/utils/errors.d.ts.map +1 -0
  141. package/dist/utils/errors.js +24 -0
  142. package/dist/utils/formatters.d.ts +145 -0
  143. package/dist/utils/formatters.d.ts.map +1 -0
  144. package/dist/utils/{paginationUtils.js → formatters.js} +54 -3
  145. package/dist/utils/index.d.ts +23 -0
  146. package/dist/utils/index.d.ts.map +1 -0
  147. package/dist/utils/index.js +69 -31
  148. package/dist/utils/indexes.d.ts +270 -0
  149. package/dist/utils/indexes.d.ts.map +1 -0
  150. package/dist/utils/indexes.js +526 -0
  151. package/dist/utils/logger.d.ts +24 -0
  152. package/dist/utils/logger.d.ts.map +1 -0
  153. package/dist/utils/operationUtils.d.ts +124 -0
  154. package/dist/utils/operationUtils.d.ts.map +1 -0
  155. package/dist/utils/operationUtils.js +175 -0
  156. package/dist/utils/parallelUtils.d.ts +72 -0
  157. package/dist/utils/parallelUtils.d.ts.map +1 -0
  158. package/dist/utils/parallelUtils.js +169 -0
  159. package/dist/utils/schemas.d.ts +374 -0
  160. package/dist/utils/schemas.d.ts.map +1 -0
  161. package/dist/utils/schemas.js +302 -2
  162. package/dist/utils/searchAlgorithms.d.ts +99 -0
  163. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  164. package/dist/utils/searchAlgorithms.js +167 -0
  165. package/dist/utils/searchCache.d.ts +108 -0
  166. package/dist/utils/searchCache.d.ts.map +1 -0
  167. package/dist/utils/taskScheduler.d.ts +290 -0
  168. package/dist/utils/taskScheduler.d.ts.map +1 -0
  169. package/dist/utils/taskScheduler.js +466 -0
  170. package/dist/workers/index.d.ts +12 -0
  171. package/dist/workers/index.d.ts.map +1 -0
  172. package/dist/workers/index.js +9 -0
  173. package/dist/workers/levenshteinWorker.d.ts +60 -0
  174. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  175. package/dist/workers/levenshteinWorker.js +98 -0
  176. package/package.json +17 -4
  177. package/dist/__tests__/edge-cases/edge-cases.test.js +0 -406
  178. package/dist/__tests__/integration/workflows.test.js +0 -449
  179. package/dist/__tests__/performance/benchmarks.test.js +0 -413
  180. package/dist/__tests__/unit/core/EntityManager.test.js +0 -334
  181. package/dist/__tests__/unit/core/GraphStorage.test.js +0 -205
  182. package/dist/__tests__/unit/core/RelationManager.test.js +0 -274
  183. package/dist/__tests__/unit/features/CompressionManager.test.js +0 -350
  184. package/dist/__tests__/unit/search/BasicSearch.test.js +0 -311
  185. package/dist/__tests__/unit/search/BooleanSearch.test.js +0 -432
  186. package/dist/__tests__/unit/search/FuzzySearch.test.js +0 -448
  187. package/dist/__tests__/unit/search/RankedSearch.test.js +0 -379
  188. package/dist/__tests__/unit/utils/levenshtein.test.js +0 -77
  189. package/dist/core/KnowledgeGraphManager.js +0 -423
  190. package/dist/features/BackupManager.js +0 -311
  191. package/dist/features/ExportManager.js +0 -305
  192. package/dist/features/ImportExportManager.js +0 -50
  193. package/dist/features/ImportManager.js +0 -328
  194. package/dist/types/analytics.types.js +0 -6
  195. package/dist/types/entity.types.js +0 -7
  196. package/dist/types/import-export.types.js +0 -7
  197. package/dist/types/search.types.js +0 -7
  198. package/dist/types/tag.types.js +0 -6
  199. package/dist/utils/dateUtils.js +0 -89
  200. package/dist/utils/filterUtils.js +0 -155
  201. package/dist/utils/levenshtein.js +0 -62
  202. package/dist/utils/pathUtils.js +0 -115
  203. package/dist/utils/responseFormatter.js +0 -55
  204. package/dist/utils/tagUtils.js +0 -107
  205. package/dist/utils/tfidf.js +0 -90
  206. package/dist/utils/validationHelper.js +0 -99
  207. package/dist/utils/validationUtils.js +0 -109
@@ -2,12 +2,16 @@
2
2
  * Graph Storage
3
3
  *
4
4
  * Handles file I/O operations for the knowledge graph using JSONL format.
5
- * Provides persistence layer abstraction for graph data.
5
+ * Implements IGraphStorage interface for storage abstraction.
6
6
  *
7
7
  * @module core/GraphStorage
8
8
  */
9
9
  import { promises as fs } from 'fs';
10
+ import { Mutex } from 'async-mutex';
10
11
  import { clearAllSearchCaches } from '../utils/searchCache.js';
12
+ import { NameIndex, TypeIndex, LowercaseCache, RelationIndex, ObservationIndex } from '../utils/indexes.js';
13
+ import { BatchTransaction } from './TransactionManager.js';
14
+ import { GraphEventEmitter } from './GraphEventEmitter.js';
11
15
  /**
12
16
  * GraphStorage manages persistence of the knowledge graph to disk.
13
17
  *
@@ -27,11 +31,59 @@ import { clearAllSearchCaches } from '../utils/searchCache.js';
27
31
  */
28
32
  export class GraphStorage {
29
33
  memoryFilePath;
34
+ /**
35
+ * Mutex for thread-safe access to storage operations.
36
+ * Prevents concurrent writes from corrupting the file or cache.
37
+ */
38
+ mutex = new Mutex();
30
39
  /**
31
40
  * In-memory cache of the knowledge graph.
32
41
  * Null when cache is empty or invalidated.
33
42
  */
34
43
  cache = null;
44
+ /**
45
+ * Number of pending append operations since last compaction.
46
+ * Used to trigger automatic compaction when threshold is reached.
47
+ */
48
+ pendingAppends = 0;
49
+ /**
50
+ * Dynamic threshold for automatic compaction.
51
+ *
52
+ * Returns the larger of 100 or 10% of the current entity count.
53
+ * This scales with graph size to avoid too-frequent compaction on large graphs
54
+ * while maintaining a reasonable minimum for small graphs.
55
+ *
56
+ * @returns Compaction threshold value
57
+ */
58
+ get compactionThreshold() {
59
+ return Math.max(100, Math.floor((this.cache?.entities.length ?? 0) * 0.1));
60
+ }
61
+ /**
62
+ * O(1) entity lookup by name.
63
+ */
64
+ nameIndex = new NameIndex();
65
+ /**
66
+ * O(1) entity lookup by type.
67
+ */
68
+ typeIndex = new TypeIndex();
69
+ /**
70
+ * Pre-computed lowercase strings for search optimization.
71
+ */
72
+ lowercaseCache = new LowercaseCache();
73
+ /**
74
+ * O(1) relation lookup by entity name.
75
+ */
76
+ relationIndex = new RelationIndex();
77
+ /**
78
+ * O(1) observation word lookup by entity.
79
+ * Maps words in observations to entity names.
80
+ */
81
+ observationIndex = new ObservationIndex();
82
+ /**
83
+ * Phase 10 Sprint 2: Event emitter for graph change notifications.
84
+ * Allows external systems to subscribe to graph changes.
85
+ */
86
+ eventEmitter = new GraphEventEmitter();
35
87
  /**
36
88
  * Create a new GraphStorage instance.
37
89
  *
@@ -40,32 +92,123 @@ export class GraphStorage {
40
92
  constructor(memoryFilePath) {
41
93
  this.memoryFilePath = memoryFilePath;
42
94
  }
95
+ // ==================== Phase 10 Sprint 2: Event Emitter Access ====================
43
96
  /**
44
- * Load the knowledge graph from disk.
97
+ * Get the event emitter for subscribing to graph changes.
98
+ *
99
+ * @returns GraphEventEmitter instance
100
+ *
101
+ * @example
102
+ * ```typescript
103
+ * const storage = new GraphStorage('/data/memory.jsonl');
45
104
  *
46
- * OPTIMIZED: Uses in-memory cache to avoid repeated disk reads.
47
- * Cache is populated on first load and invalidated on writes.
105
+ * // Subscribe to entity creation events
106
+ * storage.events.on('entity:created', (event) => {
107
+ * console.log(`Entity ${event.entity.name} created`);
108
+ * });
48
109
  *
49
- * Reads the JSONL file and reconstructs the graph structure.
50
- * Returns empty graph if file doesn't exist.
110
+ * // Subscribe to all events
111
+ * storage.events.onAny((event) => {
112
+ * console.log(`Graph event: ${event.type}`);
113
+ * });
114
+ * ```
115
+ */
116
+ get events() {
117
+ return this.eventEmitter;
118
+ }
119
+ // ==================== Durable File Operations ====================
120
+ /**
121
+ * Write content to file with fsync for durability.
51
122
  *
52
- * @returns Promise resolving to the loaded knowledge graph
123
+ * @param content - Content to write
124
+ */
125
+ async durableWriteFile(content) {
126
+ const fd = await fs.open(this.memoryFilePath, 'w');
127
+ try {
128
+ await fd.write(content);
129
+ await fd.sync();
130
+ }
131
+ finally {
132
+ await fd.close();
133
+ }
134
+ }
135
+ /**
136
+ * Append content to file with fsync for durability.
137
+ *
138
+ * @param content - Content to append
139
+ * @param prependNewline - Whether to prepend a newline
140
+ */
141
+ async durableAppendFile(content, prependNewline) {
142
+ const fd = await fs.open(this.memoryFilePath, 'a');
143
+ try {
144
+ const dataToWrite = prependNewline ? '\n' + content : content;
145
+ await fd.write(dataToWrite);
146
+ await fd.sync();
147
+ }
148
+ finally {
149
+ await fd.close();
150
+ }
151
+ }
152
+ /**
153
+ * Load the knowledge graph from disk (read-only access).
154
+ *
155
+ * OPTIMIZED: Returns cached reference directly without copying.
156
+ * This is O(1) regardless of graph size. For mutation operations,
157
+ * use getGraphForMutation() instead.
158
+ *
159
+ * @returns Promise resolving to read-only knowledge graph reference
53
160
  * @throws Error if file exists but cannot be read or parsed
54
161
  */
55
162
  async loadGraph() {
56
- // Return cached graph if available
163
+ // Return cached graph directly (no copying - O(1))
57
164
  if (this.cache !== null) {
58
- // Return a deep copy to prevent external mutations from affecting cache
59
- return {
60
- entities: this.cache.entities.map(e => ({ ...e })),
61
- relations: this.cache.relations.map(r => ({ ...r })),
62
- };
165
+ return this.cache;
63
166
  }
64
167
  // Cache miss - load from disk
168
+ await this.loadFromDisk();
169
+ return this.cache;
170
+ }
171
+ /**
172
+ * Get a mutable copy of the graph for write operations.
173
+ *
174
+ * Creates deep copies of entity and relation arrays to allow
175
+ * safe mutation without affecting the cached data.
176
+ *
177
+ * @returns Promise resolving to mutable knowledge graph copy
178
+ */
179
+ async getGraphForMutation() {
180
+ await this.ensureLoaded();
181
+ return {
182
+ entities: this.cache.entities.map(e => ({
183
+ ...e,
184
+ observations: [...e.observations],
185
+ tags: e.tags ? [...e.tags] : undefined,
186
+ })),
187
+ relations: this.cache.relations.map(r => ({ ...r })),
188
+ };
189
+ }
190
+ /**
191
+ * Ensure the cache is loaded from disk.
192
+ *
193
+ * @returns Promise resolving when cache is populated
194
+ */
195
+ async ensureLoaded() {
196
+ if (this.cache === null) {
197
+ await this.loadFromDisk();
198
+ }
199
+ }
200
+ /**
201
+ * Internal method to load graph from disk into cache.
202
+ */
203
+ async loadFromDisk() {
65
204
  try {
66
205
  const data = await fs.readFile(this.memoryFilePath, 'utf-8');
67
206
  const lines = data.split('\n').filter((line) => line.trim() !== '');
68
- const graph = lines.reduce((graph, line) => {
207
+ // Use Maps to deduplicate - later entries override earlier ones
208
+ // This supports append-only updates where new versions are appended
209
+ const entityMap = new Map();
210
+ const relationMap = new Map();
211
+ for (const line of lines) {
69
212
  const item = JSON.parse(line);
70
213
  if (item.type === 'entity') {
71
214
  // Add createdAt if missing for backward compatibility
@@ -74,7 +217,8 @@ export class GraphStorage {
74
217
  // Add lastModified if missing for backward compatibility
75
218
  if (!item.lastModified)
76
219
  item.lastModified = item.createdAt;
77
- graph.entities.push(item);
220
+ // Use name as key - later entries override earlier ones
221
+ entityMap.set(item.name, item);
78
222
  }
79
223
  if (item.type === 'relation') {
80
224
  // Add createdAt if missing for backward compatibility
@@ -83,32 +227,70 @@ export class GraphStorage {
83
227
  // Add lastModified if missing for backward compatibility
84
228
  if (!item.lastModified)
85
229
  item.lastModified = item.createdAt;
86
- graph.relations.push(item);
230
+ // Use composite key for relations
231
+ const key = `${item.from}:${item.to}:${item.relationType}`;
232
+ relationMap.set(key, item);
87
233
  }
88
- return graph;
89
- }, { entities: [], relations: [] });
234
+ }
235
+ // Convert maps to arrays
236
+ const graph = {
237
+ entities: Array.from(entityMap.values()),
238
+ relations: Array.from(relationMap.values()),
239
+ };
90
240
  // Populate cache
91
241
  this.cache = graph;
92
- // Return a deep copy
93
- return {
94
- entities: graph.entities.map(e => ({ ...e })),
95
- relations: graph.relations.map(r => ({ ...r })),
96
- };
242
+ // Build indexes from loaded data
243
+ this.buildEntityIndexes(graph.entities);
244
+ this.buildRelationIndex(graph.relations);
245
+ // Phase 10 Sprint 2: Emit graph:loaded event
246
+ this.eventEmitter.emitGraphLoaded(graph.entities.length, graph.relations.length);
97
247
  }
98
248
  catch (error) {
99
- // File doesn't exist - return empty graph
249
+ // File doesn't exist - create empty graph
100
250
  if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
101
- const emptyGraph = { entities: [], relations: [] };
102
- this.cache = emptyGraph;
103
- return { entities: [], relations: [] };
251
+ this.cache = { entities: [], relations: [] };
252
+ this.clearIndexes();
253
+ // Phase 10 Sprint 2: Emit graph:loaded event for empty graph
254
+ this.eventEmitter.emitGraphLoaded(0, 0);
255
+ return;
104
256
  }
105
257
  throw error;
106
258
  }
107
259
  }
260
+ /**
261
+ * Build all entity indexes from entity array.
262
+ */
263
+ buildEntityIndexes(entities) {
264
+ this.nameIndex.build(entities);
265
+ this.typeIndex.build(entities);
266
+ this.lowercaseCache.build(entities);
267
+ // Build observation index
268
+ this.observationIndex.clear();
269
+ for (const entity of entities) {
270
+ this.observationIndex.add(entity.name, entity.observations);
271
+ }
272
+ }
273
+ /**
274
+ * Build relation index from relation array.
275
+ */
276
+ buildRelationIndex(relations) {
277
+ this.relationIndex.build(relations);
278
+ }
279
+ /**
280
+ * Clear all indexes.
281
+ */
282
+ clearIndexes() {
283
+ this.nameIndex.clear();
284
+ this.typeIndex.clear();
285
+ this.lowercaseCache.clear();
286
+ this.relationIndex.clear();
287
+ this.observationIndex.clear();
288
+ }
108
289
  /**
109
290
  * Save the knowledge graph to disk.
110
291
  *
111
- * OPTIMIZED: Invalidates cache after write to ensure consistency.
292
+ * OPTIMIZED: Updates cache directly after write to avoid re-reading.
293
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
112
294
  *
113
295
  * Writes the graph to JSONL format, with one JSON object per line.
114
296
  *
@@ -117,6 +299,155 @@ export class GraphStorage {
117
299
  * @throws Error if file cannot be written
118
300
  */
119
301
  async saveGraph(graph) {
302
+ return this.mutex.runExclusive(async () => {
303
+ await this.saveGraphInternal(graph);
304
+ });
305
+ }
306
+ /**
307
+ * Append a single entity to the file (O(1) write operation).
308
+ *
309
+ * OPTIMIZED: Uses file append instead of full rewrite.
310
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
311
+ * Updates cache in-place and triggers compaction when threshold is reached.
312
+ *
313
+ * @param entity - The entity to append
314
+ * @returns Promise resolving when append is complete
315
+ */
316
+ async appendEntity(entity) {
317
+ await this.ensureLoaded();
318
+ return this.mutex.runExclusive(async () => {
319
+ const entityData = {
320
+ type: 'entity',
321
+ name: entity.name,
322
+ entityType: entity.entityType,
323
+ observations: entity.observations,
324
+ createdAt: entity.createdAt,
325
+ lastModified: entity.lastModified,
326
+ };
327
+ // Only include optional fields if they exist
328
+ if (entity.tags !== undefined)
329
+ entityData.tags = entity.tags;
330
+ if (entity.importance !== undefined)
331
+ entityData.importance = entity.importance;
332
+ if (entity.parentId !== undefined)
333
+ entityData.parentId = entity.parentId;
334
+ const line = JSON.stringify(entityData);
335
+ // Append to file with fsync for durability (write FIRST, then update cache)
336
+ try {
337
+ const stat = await fs.stat(this.memoryFilePath);
338
+ await this.durableAppendFile(line, stat.size > 0);
339
+ }
340
+ catch (error) {
341
+ // File doesn't exist - create it
342
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
343
+ await this.durableWriteFile(line);
344
+ }
345
+ else {
346
+ throw error;
347
+ }
348
+ }
349
+ // Update cache in-place (after successful file write)
350
+ this.cache.entities.push(entity);
351
+ // Update indexes
352
+ this.nameIndex.add(entity);
353
+ this.typeIndex.add(entity);
354
+ this.lowercaseCache.set(entity);
355
+ this.observationIndex.add(entity.name, entity.observations);
356
+ this.pendingAppends++;
357
+ // Clear search caches
358
+ clearAllSearchCaches();
359
+ // Phase 10 Sprint 2: Emit entity:created event
360
+ this.eventEmitter.emitEntityCreated(entity);
361
+ // Trigger compaction if threshold reached
362
+ if (this.pendingAppends >= this.compactionThreshold) {
363
+ await this.compactInternal();
364
+ }
365
+ });
366
+ }
367
+ /**
368
+ * Append a single relation to the file (O(1) write operation).
369
+ *
370
+ * OPTIMIZED: Uses file append instead of full rewrite.
371
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
372
+ * Updates cache in-place and triggers compaction when threshold is reached.
373
+ *
374
+ * @param relation - The relation to append
375
+ * @returns Promise resolving when append is complete
376
+ */
377
+ async appendRelation(relation) {
378
+ await this.ensureLoaded();
379
+ return this.mutex.runExclusive(async () => {
380
+ const line = JSON.stringify({
381
+ type: 'relation',
382
+ from: relation.from,
383
+ to: relation.to,
384
+ relationType: relation.relationType,
385
+ createdAt: relation.createdAt,
386
+ lastModified: relation.lastModified,
387
+ });
388
+ // Append to file with fsync for durability (write FIRST, then update cache)
389
+ try {
390
+ const stat = await fs.stat(this.memoryFilePath);
391
+ await this.durableAppendFile(line, stat.size > 0);
392
+ }
393
+ catch (error) {
394
+ // File doesn't exist - create it
395
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
396
+ await this.durableWriteFile(line);
397
+ }
398
+ else {
399
+ throw error;
400
+ }
401
+ }
402
+ // Update cache in-place (after successful file write)
403
+ this.cache.relations.push(relation);
404
+ // Update relation index
405
+ this.relationIndex.add(relation);
406
+ this.pendingAppends++;
407
+ // Clear search caches
408
+ clearAllSearchCaches();
409
+ // Phase 10 Sprint 2: Emit relation:created event
410
+ this.eventEmitter.emitRelationCreated(relation);
411
+ // Trigger compaction if threshold reached
412
+ if (this.pendingAppends >= this.compactionThreshold) {
413
+ await this.compactInternal();
414
+ }
415
+ });
416
+ }
417
+ /**
418
+ * Compact the file by rewriting it with only current cache contents.
419
+ *
420
+ * THREAD-SAFE: Uses mutex to prevent concurrent operations.
421
+ * Removes duplicate entries and cleans up the file.
422
+ * Resets pending appends counter.
423
+ *
424
+ * @returns Promise resolving when compaction is complete
425
+ */
426
+ async compact() {
427
+ return this.mutex.runExclusive(async () => {
428
+ await this.compactInternal();
429
+ });
430
+ }
431
+ /**
432
+ * Internal compact implementation (must be called within mutex).
433
+ *
434
+ * @returns Promise resolving when compaction is complete
435
+ */
436
+ async compactInternal() {
437
+ if (this.cache === null) {
438
+ return;
439
+ }
440
+ // Rewrite file with current cache (removes duplicates/updates)
441
+ await this.saveGraphInternal(this.cache);
442
+ this.pendingAppends = 0;
443
+ }
444
+ /**
445
+ * Internal saveGraph implementation (must be called within mutex).
446
+ *
447
+ * @param graph - The knowledge graph to save
448
+ * @returns Promise resolving when save is complete
449
+ */
450
+ async saveGraphInternal(graph) {
120
451
  const lines = [
121
452
  ...graph.entities.map(e => {
122
453
  const entityData = {
@@ -145,11 +476,116 @@ export class GraphStorage {
145
476
  lastModified: r.lastModified,
146
477
  })),
147
478
  ];
148
- await fs.writeFile(this.memoryFilePath, lines.join('\n'));
149
- // Invalidate cache to ensure next load reads fresh data
150
- this.cache = null;
479
+ await this.durableWriteFile(lines.join('\n'));
480
+ // Update cache directly with the saved graph (avoid re-reading from disk)
481
+ this.cache = graph;
482
+ // Rebuild indexes with new graph data
483
+ this.buildEntityIndexes(graph.entities);
484
+ this.buildRelationIndex(graph.relations);
485
+ // Reset pending appends since file is now clean
486
+ this.pendingAppends = 0;
151
487
  // Clear all search caches since graph data has changed
152
488
  clearAllSearchCaches();
489
+ // Phase 10 Sprint 2: Emit graph:saved event
490
+ this.eventEmitter.emitGraphSaved(graph.entities.length, graph.relations.length);
491
+ }
492
+ /**
493
+ * Get the current pending appends count.
494
+ *
495
+ * Useful for testing compaction behavior.
496
+ *
497
+ * @returns Number of pending appends since last compaction
498
+ */
499
+ getPendingAppends() {
500
+ return this.pendingAppends;
501
+ }
502
+ /**
503
+ * Update an entity in-place in the cache and append to file.
504
+ *
505
+ * OPTIMIZED: Modifies cache directly and appends updated version to file.
506
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
507
+ * Does not rewrite the entire file - compaction handles deduplication later.
508
+ *
509
+ * @param entityName - Name of the entity to update
510
+ * @param updates - Partial entity updates to apply
511
+ * @returns Promise resolving to true if entity was found and updated, false otherwise
512
+ */
513
+ async updateEntity(entityName, updates) {
514
+ await this.ensureLoaded();
515
+ return this.mutex.runExclusive(async () => {
516
+ const entityIndex = this.cache.entities.findIndex(e => e.name === entityName);
517
+ if (entityIndex === -1) {
518
+ return false;
519
+ }
520
+ const entity = this.cache.entities[entityIndex];
521
+ const oldType = entity.entityType;
522
+ const timestamp = new Date().toISOString();
523
+ // Phase 10 Sprint 2: Capture previous values for event
524
+ const previousValues = {};
525
+ for (const key of Object.keys(updates)) {
526
+ if (key in entity) {
527
+ previousValues[key] = entity[key];
528
+ }
529
+ }
530
+ // Build the updated entity data for file write BEFORE modifying cache
531
+ // This ensures cache consistency if file write fails
532
+ const updatedEntity = {
533
+ ...entity,
534
+ ...updates,
535
+ lastModified: timestamp,
536
+ };
537
+ const entityData = {
538
+ type: 'entity',
539
+ name: updatedEntity.name,
540
+ entityType: updatedEntity.entityType,
541
+ observations: updatedEntity.observations,
542
+ createdAt: updatedEntity.createdAt,
543
+ lastModified: updatedEntity.lastModified,
544
+ };
545
+ if (updatedEntity.tags !== undefined)
546
+ entityData.tags = updatedEntity.tags;
547
+ if (updatedEntity.importance !== undefined)
548
+ entityData.importance = updatedEntity.importance;
549
+ if (updatedEntity.parentId !== undefined)
550
+ entityData.parentId = updatedEntity.parentId;
551
+ const line = JSON.stringify(entityData);
552
+ // Write to file FIRST with durability - if this fails, cache remains consistent
553
+ try {
554
+ const stat = await fs.stat(this.memoryFilePath);
555
+ await this.durableAppendFile(line, stat.size > 0);
556
+ }
557
+ catch (error) {
558
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
559
+ await this.durableWriteFile(line);
560
+ }
561
+ else {
562
+ throw error;
563
+ }
564
+ }
565
+ // File write succeeded - NOW update cache in-place
566
+ Object.assign(entity, updates);
567
+ entity.lastModified = timestamp;
568
+ // Update indexes
569
+ this.nameIndex.add(entity); // Update reference
570
+ if (updates.entityType && updates.entityType !== oldType) {
571
+ this.typeIndex.updateType(entityName, oldType, updates.entityType);
572
+ }
573
+ this.lowercaseCache.set(entity); // Recompute lowercase
574
+ if (updates.observations) {
575
+ this.observationIndex.remove(entityName); // Remove old observations
576
+ this.observationIndex.add(entityName, entity.observations); // Add new observations
577
+ }
578
+ this.pendingAppends++;
579
+ // Clear search caches
580
+ clearAllSearchCaches();
581
+ // Phase 10 Sprint 2: Emit entity:updated event
582
+ this.eventEmitter.emitEntityUpdated(entityName, updates, previousValues);
583
+ // Trigger compaction if threshold reached
584
+ if (this.pendingAppends >= this.compactionThreshold) {
585
+ await this.compactInternal();
586
+ }
587
+ return true;
588
+ });
153
589
  }
154
590
  /**
155
591
  * Manually clear the cache.
@@ -160,6 +596,7 @@ export class GraphStorage {
160
596
  */
161
597
  clearCache() {
162
598
  this.cache = null;
599
+ this.clearIndexes();
163
600
  }
164
601
  /**
165
602
  * Get the file path being used for storage.
@@ -169,4 +606,179 @@ export class GraphStorage {
169
606
  getFilePath() {
170
607
  return this.memoryFilePath;
171
608
  }
609
+ // ==================== Index Accessors ====================
610
+ /**
611
+ * Get an entity by name in O(1) time.
612
+ *
613
+ * OPTIMIZED: Uses NameIndex for constant-time lookup.
614
+ *
615
+ * @param name - Entity name to look up
616
+ * @returns Entity if found, undefined otherwise
617
+ */
618
+ getEntityByName(name) {
619
+ return this.nameIndex.get(name);
620
+ }
621
+ /**
622
+ * Check if an entity exists by name in O(1) time.
623
+ *
624
+ * @param name - Entity name to check
625
+ * @returns True if entity exists
626
+ */
627
+ hasEntity(name) {
628
+ return this.nameIndex.has(name);
629
+ }
630
+ /**
631
+ * Get all entities of a given type in O(1) time.
632
+ *
633
+ * OPTIMIZED: Uses TypeIndex for constant-time lookup of entity names,
634
+ * then uses NameIndex for O(1) entity retrieval.
635
+ *
636
+ * @param entityType - Entity type to filter by (case-insensitive)
637
+ * @returns Array of entities with the given type
638
+ */
639
+ getEntitiesByType(entityType) {
640
+ const names = this.typeIndex.getNames(entityType);
641
+ const entities = [];
642
+ for (const name of names) {
643
+ const entity = this.nameIndex.get(name);
644
+ if (entity) {
645
+ entities.push(entity);
646
+ }
647
+ }
648
+ return entities;
649
+ }
650
+ /**
651
+ * Get pre-computed lowercase data for an entity.
652
+ *
653
+ * OPTIMIZED: Avoids repeated toLowerCase() calls during search.
654
+ *
655
+ * @param entityName - Entity name to get lowercase data for
656
+ * @returns LowercaseData if entity exists, undefined otherwise
657
+ */
658
+ getLowercased(entityName) {
659
+ return this.lowercaseCache.get(entityName);
660
+ }
661
+ /**
662
+ * Get all unique entity types in the graph.
663
+ *
664
+ * @returns Array of unique entity types (lowercase)
665
+ */
666
+ getEntityTypes() {
667
+ return this.typeIndex.getTypes();
668
+ }
669
+ // ==================== Relation Index Accessors ====================
670
+ /**
671
+ * Get all relations where the entity is the source (outgoing relations) in O(1) time.
672
+ *
673
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
674
+ *
675
+ * @param entityName - Entity name to look up outgoing relations for
676
+ * @returns Array of relations where entity is the source
677
+ */
678
+ getRelationsFrom(entityName) {
679
+ return this.relationIndex.getRelationsFrom(entityName);
680
+ }
681
+ /**
682
+ * Get all relations where the entity is the target (incoming relations) in O(1) time.
683
+ *
684
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
685
+ *
686
+ * @param entityName - Entity name to look up incoming relations for
687
+ * @returns Array of relations where entity is the target
688
+ */
689
+ getRelationsTo(entityName) {
690
+ return this.relationIndex.getRelationsTo(entityName);
691
+ }
692
+ /**
693
+ * Get all relations involving the entity (both incoming and outgoing) in O(1) time.
694
+ *
695
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
696
+ *
697
+ * @param entityName - Entity name to look up all relations for
698
+ * @returns Array of all relations involving the entity
699
+ */
700
+ getRelationsFor(entityName) {
701
+ return this.relationIndex.getRelationsFor(entityName);
702
+ }
703
+ /**
704
+ * Check if an entity has any relations.
705
+ *
706
+ * @param entityName - Entity name to check
707
+ * @returns True if entity has any relations
708
+ */
709
+ hasRelations(entityName) {
710
+ return this.relationIndex.hasRelations(entityName);
711
+ }
712
+ // ==================== Observation Index Accessors ====================
713
+ /**
714
+ * Get entities that have observations containing the given word.
715
+ * Uses the observation index for O(1) lookup.
716
+ *
717
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookup instead of
718
+ * linear scan through all entities and their observations.
719
+ *
720
+ * @param word - Word to search for in observations
721
+ * @returns Set of entity names
722
+ */
723
+ getEntitiesByObservationWord(word) {
724
+ return this.observationIndex.getEntitiesWithWord(word);
725
+ }
726
+ /**
727
+ * Get entities that have observations containing ANY of the given words (union).
728
+ * Uses the observation index for O(1) lookup per word.
729
+ *
730
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookups.
731
+ *
732
+ * @param words - Array of words to search for
733
+ * @returns Set of entity names containing any of the words
734
+ */
735
+ getEntitiesByAnyObservationWord(words) {
736
+ return this.observationIndex.getEntitiesWithAnyWord(words);
737
+ }
738
+ /**
739
+ * Get entities that have observations containing ALL of the given words (intersection).
740
+ * Uses the observation index for O(1) lookup per word.
741
+ *
742
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookups and set intersection.
743
+ *
744
+ * @param words - Array of words that must all be present
745
+ * @returns Set of entity names containing all of the words
746
+ */
747
+ getEntitiesByAllObservationWords(words) {
748
+ return this.observationIndex.getEntitiesWithAllWords(words);
749
+ }
750
+ /**
751
+ * Get statistics about the observation index.
752
+ *
753
+ * @returns Object with wordCount and entityCount
754
+ */
755
+ getObservationIndexStats() {
756
+ return this.observationIndex.getStats();
757
+ }
758
+ // ==================== Phase 10 Sprint 1: Transaction Factory ====================
759
+ /**
760
+ * Create a new batch transaction for atomic operations.
761
+ *
762
+ * Returns a BatchTransaction instance that can be used to queue multiple
763
+ * operations and execute them atomically with a single save operation.
764
+ *
765
+ * @returns A new BatchTransaction instance
766
+ *
767
+ * @example
768
+ * ```typescript
769
+ * const storage = new GraphStorage('/data/memory.jsonl');
770
+ *
771
+ * // Create and execute a batch transaction
772
+ * const result = await storage.transaction()
773
+ * .createEntity({ name: 'Alice', entityType: 'person', observations: ['Developer'] })
774
+ * .createEntity({ name: 'Bob', entityType: 'person', observations: ['Designer'] })
775
+ * .createRelation({ from: 'Alice', to: 'Bob', relationType: 'knows' })
776
+ * .execute();
777
+ *
778
+ * console.log(`Batch completed: ${result.operationsExecuted} operations`);
779
+ * ```
780
+ */
781
+ transaction() {
782
+ return new BatchTransaction(this);
783
+ }
172
784
  }