@danielsimonjr/memory-mcp 0.48.0 → 9.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +2000 -194
  3. package/dist/__tests__/file-path.test.js +7 -11
  4. package/dist/__tests__/knowledge-graph.test.js +3 -8
  5. package/dist/core/EntityManager.d.ts +266 -0
  6. package/dist/core/EntityManager.d.ts.map +1 -0
  7. package/dist/core/EntityManager.js +89 -137
  8. package/dist/core/GraphEventEmitter.d.ts +202 -0
  9. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  10. package/dist/core/GraphEventEmitter.js +346 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +644 -31
  14. package/dist/core/GraphTraversal.d.ts +141 -0
  15. package/dist/core/GraphTraversal.d.ts.map +1 -0
  16. package/dist/core/GraphTraversal.js +573 -0
  17. package/dist/core/HierarchyManager.d.ts +111 -0
  18. package/dist/core/HierarchyManager.d.ts.map +1 -0
  19. package/dist/{features → core}/HierarchyManager.js +14 -9
  20. package/dist/core/ManagerContext.d.ts +72 -0
  21. package/dist/core/ManagerContext.d.ts.map +1 -0
  22. package/dist/core/ManagerContext.js +118 -0
  23. package/dist/core/ObservationManager.d.ts +85 -0
  24. package/dist/core/ObservationManager.d.ts.map +1 -0
  25. package/dist/core/ObservationManager.js +51 -57
  26. package/dist/core/RelationManager.d.ts +131 -0
  27. package/dist/core/RelationManager.d.ts.map +1 -0
  28. package/dist/core/RelationManager.js +31 -7
  29. package/dist/core/SQLiteStorage.d.ts +354 -0
  30. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  31. package/dist/core/SQLiteStorage.js +918 -0
  32. package/dist/core/StorageFactory.d.ts +45 -0
  33. package/dist/core/StorageFactory.d.ts.map +1 -0
  34. package/dist/core/StorageFactory.js +64 -0
  35. package/dist/core/TransactionManager.d.ts +464 -0
  36. package/dist/core/TransactionManager.d.ts.map +1 -0
  37. package/dist/core/TransactionManager.js +493 -14
  38. package/dist/core/index.d.ts +17 -0
  39. package/dist/core/index.d.ts.map +1 -0
  40. package/dist/core/index.js +12 -2
  41. package/dist/features/AnalyticsManager.d.ts +44 -0
  42. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  43. package/dist/features/AnalyticsManager.js +3 -2
  44. package/dist/features/ArchiveManager.d.ts +133 -0
  45. package/dist/features/ArchiveManager.d.ts.map +1 -0
  46. package/dist/features/ArchiveManager.js +221 -14
  47. package/dist/features/CompressionManager.d.ts +117 -0
  48. package/dist/features/CompressionManager.d.ts.map +1 -0
  49. package/dist/features/CompressionManager.js +189 -20
  50. package/dist/features/IOManager.d.ts +225 -0
  51. package/dist/features/IOManager.d.ts.map +1 -0
  52. package/dist/features/IOManager.js +1092 -0
  53. package/dist/features/StreamingExporter.d.ts +128 -0
  54. package/dist/features/StreamingExporter.d.ts.map +1 -0
  55. package/dist/features/StreamingExporter.js +211 -0
  56. package/dist/features/TagManager.d.ts +147 -0
  57. package/dist/features/TagManager.d.ts.map +1 -0
  58. package/dist/features/index.d.ts +12 -0
  59. package/dist/features/index.d.ts.map +1 -0
  60. package/dist/features/index.js +5 -6
  61. package/dist/index.d.ts +9 -0
  62. package/dist/index.d.ts.map +1 -0
  63. package/dist/index.js +10 -10
  64. package/dist/memory.jsonl +1 -26
  65. package/dist/search/BasicSearch.d.ts +51 -0
  66. package/dist/search/BasicSearch.d.ts.map +1 -0
  67. package/dist/search/BasicSearch.js +9 -3
  68. package/dist/search/BooleanSearch.d.ts +98 -0
  69. package/dist/search/BooleanSearch.d.ts.map +1 -0
  70. package/dist/search/BooleanSearch.js +156 -9
  71. package/dist/search/EmbeddingService.d.ts +178 -0
  72. package/dist/search/EmbeddingService.d.ts.map +1 -0
  73. package/dist/search/EmbeddingService.js +358 -0
  74. package/dist/search/FuzzySearch.d.ts +118 -0
  75. package/dist/search/FuzzySearch.d.ts.map +1 -0
  76. package/dist/search/FuzzySearch.js +241 -25
  77. package/dist/search/QueryCostEstimator.d.ts +111 -0
  78. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  79. package/dist/search/QueryCostEstimator.js +355 -0
  80. package/dist/search/RankedSearch.d.ts +71 -0
  81. package/dist/search/RankedSearch.d.ts.map +1 -0
  82. package/dist/search/RankedSearch.js +54 -6
  83. package/dist/search/SavedSearchManager.d.ts +79 -0
  84. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  85. package/dist/search/SavedSearchManager.js +3 -2
  86. package/dist/search/SearchFilterChain.d.ts +120 -0
  87. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  88. package/dist/search/SearchFilterChain.js +2 -4
  89. package/dist/search/SearchManager.d.ts +326 -0
  90. package/dist/search/SearchManager.d.ts.map +1 -0
  91. package/dist/search/SearchManager.js +148 -0
  92. package/dist/search/SearchSuggestions.d.ts +27 -0
  93. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  94. package/dist/search/SearchSuggestions.js +1 -1
  95. package/dist/search/SemanticSearch.d.ts +149 -0
  96. package/dist/search/SemanticSearch.d.ts.map +1 -0
  97. package/dist/search/SemanticSearch.js +323 -0
  98. package/dist/search/TFIDFEventSync.d.ts +85 -0
  99. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  100. package/dist/search/TFIDFEventSync.js +133 -0
  101. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  102. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  103. package/dist/search/TFIDFIndexManager.js +232 -17
  104. package/dist/search/VectorStore.d.ts +235 -0
  105. package/dist/search/VectorStore.d.ts.map +1 -0
  106. package/dist/search/VectorStore.js +311 -0
  107. package/dist/search/index.d.ts +21 -0
  108. package/dist/search/index.d.ts.map +1 -0
  109. package/dist/search/index.js +12 -0
  110. package/dist/server/MCPServer.d.ts +21 -0
  111. package/dist/server/MCPServer.d.ts.map +1 -0
  112. package/dist/server/MCPServer.js +4 -4
  113. package/dist/server/responseCompressor.d.ts +94 -0
  114. package/dist/server/responseCompressor.d.ts.map +1 -0
  115. package/dist/server/responseCompressor.js +127 -0
  116. package/dist/server/toolDefinitions.d.ts +27 -0
  117. package/dist/server/toolDefinitions.d.ts.map +1 -0
  118. package/dist/server/toolDefinitions.js +188 -17
  119. package/dist/server/toolHandlers.d.ts +41 -0
  120. package/dist/server/toolHandlers.d.ts.map +1 -0
  121. package/dist/server/toolHandlers.js +469 -75
  122. package/dist/types/index.d.ts +13 -0
  123. package/dist/types/index.d.ts.map +1 -0
  124. package/dist/types/index.js +1 -1
  125. package/dist/types/types.d.ts +1654 -0
  126. package/dist/types/types.d.ts.map +1 -0
  127. package/dist/types/types.js +9 -0
  128. package/dist/utils/compressedCache.d.ts +192 -0
  129. package/dist/utils/compressedCache.d.ts.map +1 -0
  130. package/dist/utils/compressedCache.js +309 -0
  131. package/dist/utils/compressionUtil.d.ts +214 -0
  132. package/dist/utils/compressionUtil.d.ts.map +1 -0
  133. package/dist/utils/compressionUtil.js +247 -0
  134. package/dist/utils/constants.d.ts +245 -0
  135. package/dist/utils/constants.d.ts.map +1 -0
  136. package/dist/utils/constants.js +124 -0
  137. package/dist/utils/entityUtils.d.ts +354 -0
  138. package/dist/utils/entityUtils.d.ts.map +1 -0
  139. package/dist/utils/entityUtils.js +511 -4
  140. package/dist/utils/errors.d.ts +95 -0
  141. package/dist/utils/errors.d.ts.map +1 -0
  142. package/dist/utils/errors.js +24 -0
  143. package/dist/utils/formatters.d.ts +145 -0
  144. package/dist/utils/formatters.d.ts.map +1 -0
  145. package/dist/utils/{paginationUtils.js → formatters.js} +54 -3
  146. package/dist/utils/index.d.ts +23 -0
  147. package/dist/utils/index.d.ts.map +1 -0
  148. package/dist/utils/index.js +71 -31
  149. package/dist/utils/indexes.d.ts +270 -0
  150. package/dist/utils/indexes.d.ts.map +1 -0
  151. package/dist/utils/indexes.js +526 -0
  152. package/dist/utils/logger.d.ts +24 -0
  153. package/dist/utils/logger.d.ts.map +1 -0
  154. package/dist/utils/operationUtils.d.ts +124 -0
  155. package/dist/utils/operationUtils.d.ts.map +1 -0
  156. package/dist/utils/operationUtils.js +175 -0
  157. package/dist/utils/parallelUtils.d.ts +76 -0
  158. package/dist/utils/parallelUtils.d.ts.map +1 -0
  159. package/dist/utils/parallelUtils.js +191 -0
  160. package/dist/utils/schemas.d.ts +374 -0
  161. package/dist/utils/schemas.d.ts.map +1 -0
  162. package/dist/utils/schemas.js +307 -7
  163. package/dist/utils/searchAlgorithms.d.ts +99 -0
  164. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  165. package/dist/utils/searchAlgorithms.js +167 -0
  166. package/dist/utils/searchCache.d.ts +108 -0
  167. package/dist/utils/searchCache.d.ts.map +1 -0
  168. package/dist/utils/taskScheduler.d.ts +294 -0
  169. package/dist/utils/taskScheduler.d.ts.map +1 -0
  170. package/dist/utils/taskScheduler.js +486 -0
  171. package/dist/workers/index.d.ts +12 -0
  172. package/dist/workers/index.d.ts.map +1 -0
  173. package/dist/workers/index.js +9 -0
  174. package/dist/workers/levenshteinWorker.d.ts +60 -0
  175. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  176. package/dist/workers/levenshteinWorker.js +98 -0
  177. package/package.json +17 -4
  178. package/dist/__tests__/edge-cases/edge-cases.test.js +0 -406
  179. package/dist/__tests__/integration/workflows.test.js +0 -449
  180. package/dist/__tests__/performance/benchmarks.test.js +0 -413
  181. package/dist/__tests__/unit/core/EntityManager.test.js +0 -334
  182. package/dist/__tests__/unit/core/GraphStorage.test.js +0 -205
  183. package/dist/__tests__/unit/core/RelationManager.test.js +0 -274
  184. package/dist/__tests__/unit/features/CompressionManager.test.js +0 -350
  185. package/dist/__tests__/unit/search/BasicSearch.test.js +0 -311
  186. package/dist/__tests__/unit/search/BooleanSearch.test.js +0 -432
  187. package/dist/__tests__/unit/search/FuzzySearch.test.js +0 -448
  188. package/dist/__tests__/unit/search/RankedSearch.test.js +0 -379
  189. package/dist/__tests__/unit/utils/levenshtein.test.js +0 -77
  190. package/dist/core/KnowledgeGraphManager.js +0 -423
  191. package/dist/features/BackupManager.js +0 -311
  192. package/dist/features/ExportManager.js +0 -305
  193. package/dist/features/ImportExportManager.js +0 -50
  194. package/dist/features/ImportManager.js +0 -328
  195. package/dist/memory-saved-searches.jsonl +0 -0
  196. package/dist/memory-tag-aliases.jsonl +0 -0
  197. package/dist/types/analytics.types.js +0 -6
  198. package/dist/types/entity.types.js +0 -7
  199. package/dist/types/import-export.types.js +0 -7
  200. package/dist/types/search.types.js +0 -7
  201. package/dist/types/tag.types.js +0 -6
  202. package/dist/utils/dateUtils.js +0 -89
  203. package/dist/utils/filterUtils.js +0 -155
  204. package/dist/utils/levenshtein.js +0 -62
  205. package/dist/utils/pathUtils.js +0 -115
  206. package/dist/utils/responseFormatter.js +0 -55
  207. package/dist/utils/tagUtils.js +0 -107
  208. package/dist/utils/tfidf.js +0 -90
  209. package/dist/utils/validationHelper.js +0 -99
  210. package/dist/utils/validationUtils.js +0 -109
@@ -2,12 +2,17 @@
2
2
  * Graph Storage
3
3
  *
4
4
  * Handles file I/O operations for the knowledge graph using JSONL format.
5
- * Provides persistence layer abstraction for graph data.
5
+ * Implements IGraphStorage interface for storage abstraction.
6
6
  *
7
7
  * @module core/GraphStorage
8
8
  */
9
9
  import { promises as fs } from 'fs';
10
+ import { Mutex } from 'async-mutex';
10
11
  import { clearAllSearchCaches } from '../utils/searchCache.js';
12
+ import { NameIndex, TypeIndex, LowercaseCache, RelationIndex, ObservationIndex } from '../utils/indexes.js';
13
+ import { sanitizeObject } from '../utils/index.js';
14
+ import { BatchTransaction } from './TransactionManager.js';
15
+ import { GraphEventEmitter } from './GraphEventEmitter.js';
11
16
  /**
12
17
  * GraphStorage manages persistence of the knowledge graph to disk.
13
18
  *
@@ -27,11 +32,59 @@ import { clearAllSearchCaches } from '../utils/searchCache.js';
27
32
  */
28
33
  export class GraphStorage {
29
34
  memoryFilePath;
35
+ /**
36
+ * Mutex for thread-safe access to storage operations.
37
+ * Prevents concurrent writes from corrupting the file or cache.
38
+ */
39
+ mutex = new Mutex();
30
40
  /**
31
41
  * In-memory cache of the knowledge graph.
32
42
  * Null when cache is empty or invalidated.
33
43
  */
34
44
  cache = null;
45
+ /**
46
+ * Number of pending append operations since last compaction.
47
+ * Used to trigger automatic compaction when threshold is reached.
48
+ */
49
+ pendingAppends = 0;
50
+ /**
51
+ * Dynamic threshold for automatic compaction.
52
+ *
53
+ * Returns the larger of 100 or 10% of the current entity count.
54
+ * This scales with graph size to avoid too-frequent compaction on large graphs
55
+ * while maintaining a reasonable minimum for small graphs.
56
+ *
57
+ * @returns Compaction threshold value
58
+ */
59
+ get compactionThreshold() {
60
+ return Math.max(100, Math.floor((this.cache?.entities.length ?? 0) * 0.1));
61
+ }
62
+ /**
63
+ * O(1) entity lookup by name.
64
+ */
65
+ nameIndex = new NameIndex();
66
+ /**
67
+ * O(1) entity lookup by type.
68
+ */
69
+ typeIndex = new TypeIndex();
70
+ /**
71
+ * Pre-computed lowercase strings for search optimization.
72
+ */
73
+ lowercaseCache = new LowercaseCache();
74
+ /**
75
+ * O(1) relation lookup by entity name.
76
+ */
77
+ relationIndex = new RelationIndex();
78
+ /**
79
+ * O(1) observation word lookup by entity.
80
+ * Maps words in observations to entity names.
81
+ */
82
+ observationIndex = new ObservationIndex();
83
+ /**
84
+ * Phase 10 Sprint 2: Event emitter for graph change notifications.
85
+ * Allows external systems to subscribe to graph changes.
86
+ */
87
+ eventEmitter = new GraphEventEmitter();
35
88
  /**
36
89
  * Create a new GraphStorage instance.
37
90
  *
@@ -40,32 +93,123 @@ export class GraphStorage {
40
93
  constructor(memoryFilePath) {
41
94
  this.memoryFilePath = memoryFilePath;
42
95
  }
96
+ // ==================== Phase 10 Sprint 2: Event Emitter Access ====================
43
97
  /**
44
- * Load the knowledge graph from disk.
98
+ * Get the event emitter for subscribing to graph changes.
99
+ *
100
+ * @returns GraphEventEmitter instance
101
+ *
102
+ * @example
103
+ * ```typescript
104
+ * const storage = new GraphStorage('/data/memory.jsonl');
45
105
  *
46
- * OPTIMIZED: Uses in-memory cache to avoid repeated disk reads.
47
- * Cache is populated on first load and invalidated on writes.
106
+ * // Subscribe to entity creation events
107
+ * storage.events.on('entity:created', (event) => {
108
+ * console.log(`Entity ${event.entity.name} created`);
109
+ * });
48
110
  *
49
- * Reads the JSONL file and reconstructs the graph structure.
50
- * Returns empty graph if file doesn't exist.
111
+ * // Subscribe to all events
112
+ * storage.events.onAny((event) => {
113
+ * console.log(`Graph event: ${event.type}`);
114
+ * });
115
+ * ```
116
+ */
117
+ get events() {
118
+ return this.eventEmitter;
119
+ }
120
+ // ==================== Durable File Operations ====================
121
+ /**
122
+ * Write content to file with fsync for durability.
51
123
  *
52
- * @returns Promise resolving to the loaded knowledge graph
124
+ * @param content - Content to write
125
+ */
126
+ async durableWriteFile(content) {
127
+ const fd = await fs.open(this.memoryFilePath, 'w');
128
+ try {
129
+ await fd.write(content);
130
+ await fd.sync();
131
+ }
132
+ finally {
133
+ await fd.close();
134
+ }
135
+ }
136
+ /**
137
+ * Append content to file with fsync for durability.
138
+ *
139
+ * @param content - Content to append
140
+ * @param prependNewline - Whether to prepend a newline
141
+ */
142
+ async durableAppendFile(content, prependNewline) {
143
+ const fd = await fs.open(this.memoryFilePath, 'a');
144
+ try {
145
+ const dataToWrite = prependNewline ? '\n' + content : content;
146
+ await fd.write(dataToWrite);
147
+ await fd.sync();
148
+ }
149
+ finally {
150
+ await fd.close();
151
+ }
152
+ }
153
+ /**
154
+ * Load the knowledge graph from disk (read-only access).
155
+ *
156
+ * OPTIMIZED: Returns cached reference directly without copying.
157
+ * This is O(1) regardless of graph size. For mutation operations,
158
+ * use getGraphForMutation() instead.
159
+ *
160
+ * @returns Promise resolving to read-only knowledge graph reference
53
161
  * @throws Error if file exists but cannot be read or parsed
54
162
  */
55
163
  async loadGraph() {
56
- // Return cached graph if available
164
+ // Return cached graph directly (no copying - O(1))
57
165
  if (this.cache !== null) {
58
- // Return a deep copy to prevent external mutations from affecting cache
59
- return {
60
- entities: this.cache.entities.map(e => ({ ...e })),
61
- relations: this.cache.relations.map(r => ({ ...r })),
62
- };
166
+ return this.cache;
63
167
  }
64
168
  // Cache miss - load from disk
169
+ await this.loadFromDisk();
170
+ return this.cache;
171
+ }
172
+ /**
173
+ * Get a mutable copy of the graph for write operations.
174
+ *
175
+ * Creates deep copies of entity and relation arrays to allow
176
+ * safe mutation without affecting the cached data.
177
+ *
178
+ * @returns Promise resolving to mutable knowledge graph copy
179
+ */
180
+ async getGraphForMutation() {
181
+ await this.ensureLoaded();
182
+ return {
183
+ entities: this.cache.entities.map(e => ({
184
+ ...e,
185
+ observations: [...e.observations],
186
+ tags: e.tags ? [...e.tags] : undefined,
187
+ })),
188
+ relations: this.cache.relations.map(r => ({ ...r })),
189
+ };
190
+ }
191
+ /**
192
+ * Ensure the cache is loaded from disk.
193
+ *
194
+ * @returns Promise resolving when cache is populated
195
+ */
196
+ async ensureLoaded() {
197
+ if (this.cache === null) {
198
+ await this.loadFromDisk();
199
+ }
200
+ }
201
+ /**
202
+ * Internal method to load graph from disk into cache.
203
+ */
204
+ async loadFromDisk() {
65
205
  try {
66
206
  const data = await fs.readFile(this.memoryFilePath, 'utf-8');
67
207
  const lines = data.split('\n').filter((line) => line.trim() !== '');
68
- const graph = lines.reduce((graph, line) => {
208
+ // Use Maps to deduplicate - later entries override earlier ones
209
+ // This supports append-only updates where new versions are appended
210
+ const entityMap = new Map();
211
+ const relationMap = new Map();
212
+ for (const line of lines) {
69
213
  const item = JSON.parse(line);
70
214
  if (item.type === 'entity') {
71
215
  // Add createdAt if missing for backward compatibility
@@ -74,7 +218,8 @@ export class GraphStorage {
74
218
  // Add lastModified if missing for backward compatibility
75
219
  if (!item.lastModified)
76
220
  item.lastModified = item.createdAt;
77
- graph.entities.push(item);
221
+ // Use name as key - later entries override earlier ones
222
+ entityMap.set(item.name, item);
78
223
  }
79
224
  if (item.type === 'relation') {
80
225
  // Add createdAt if missing for backward compatibility
@@ -83,32 +228,70 @@ export class GraphStorage {
83
228
  // Add lastModified if missing for backward compatibility
84
229
  if (!item.lastModified)
85
230
  item.lastModified = item.createdAt;
86
- graph.relations.push(item);
231
+ // Use composite key for relations
232
+ const key = `${item.from}:${item.to}:${item.relationType}`;
233
+ relationMap.set(key, item);
87
234
  }
88
- return graph;
89
- }, { entities: [], relations: [] });
235
+ }
236
+ // Convert maps to arrays
237
+ const graph = {
238
+ entities: Array.from(entityMap.values()),
239
+ relations: Array.from(relationMap.values()),
240
+ };
90
241
  // Populate cache
91
242
  this.cache = graph;
92
- // Return a deep copy
93
- return {
94
- entities: graph.entities.map(e => ({ ...e })),
95
- relations: graph.relations.map(r => ({ ...r })),
96
- };
243
+ // Build indexes from loaded data
244
+ this.buildEntityIndexes(graph.entities);
245
+ this.buildRelationIndex(graph.relations);
246
+ // Phase 10 Sprint 2: Emit graph:loaded event
247
+ this.eventEmitter.emitGraphLoaded(graph.entities.length, graph.relations.length);
97
248
  }
98
249
  catch (error) {
99
- // File doesn't exist - return empty graph
250
+ // File doesn't exist - create empty graph
100
251
  if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
101
- const emptyGraph = { entities: [], relations: [] };
102
- this.cache = emptyGraph;
103
- return { entities: [], relations: [] };
252
+ this.cache = { entities: [], relations: [] };
253
+ this.clearIndexes();
254
+ // Phase 10 Sprint 2: Emit graph:loaded event for empty graph
255
+ this.eventEmitter.emitGraphLoaded(0, 0);
256
+ return;
104
257
  }
105
258
  throw error;
106
259
  }
107
260
  }
261
+ /**
262
+ * Build all entity indexes from entity array.
263
+ */
264
+ buildEntityIndexes(entities) {
265
+ this.nameIndex.build(entities);
266
+ this.typeIndex.build(entities);
267
+ this.lowercaseCache.build(entities);
268
+ // Build observation index
269
+ this.observationIndex.clear();
270
+ for (const entity of entities) {
271
+ this.observationIndex.add(entity.name, entity.observations);
272
+ }
273
+ }
274
+ /**
275
+ * Build relation index from relation array.
276
+ */
277
+ buildRelationIndex(relations) {
278
+ this.relationIndex.build(relations);
279
+ }
280
+ /**
281
+ * Clear all indexes.
282
+ */
283
+ clearIndexes() {
284
+ this.nameIndex.clear();
285
+ this.typeIndex.clear();
286
+ this.lowercaseCache.clear();
287
+ this.relationIndex.clear();
288
+ this.observationIndex.clear();
289
+ }
108
290
  /**
109
291
  * Save the knowledge graph to disk.
110
292
  *
111
- * OPTIMIZED: Invalidates cache after write to ensure consistency.
293
+ * OPTIMIZED: Updates cache directly after write to avoid re-reading.
294
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
112
295
  *
113
296
  * Writes the graph to JSONL format, with one JSON object per line.
114
297
  *
@@ -117,6 +300,155 @@ export class GraphStorage {
117
300
  * @throws Error if file cannot be written
118
301
  */
119
302
  async saveGraph(graph) {
303
+ return this.mutex.runExclusive(async () => {
304
+ await this.saveGraphInternal(graph);
305
+ });
306
+ }
307
+ /**
308
+ * Append a single entity to the file (O(1) write operation).
309
+ *
310
+ * OPTIMIZED: Uses file append instead of full rewrite.
311
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
312
+ * Updates cache in-place and triggers compaction when threshold is reached.
313
+ *
314
+ * @param entity - The entity to append
315
+ * @returns Promise resolving when append is complete
316
+ */
317
+ async appendEntity(entity) {
318
+ await this.ensureLoaded();
319
+ return this.mutex.runExclusive(async () => {
320
+ const entityData = {
321
+ type: 'entity',
322
+ name: entity.name,
323
+ entityType: entity.entityType,
324
+ observations: entity.observations,
325
+ createdAt: entity.createdAt,
326
+ lastModified: entity.lastModified,
327
+ };
328
+ // Only include optional fields if they exist
329
+ if (entity.tags !== undefined)
330
+ entityData.tags = entity.tags;
331
+ if (entity.importance !== undefined)
332
+ entityData.importance = entity.importance;
333
+ if (entity.parentId !== undefined)
334
+ entityData.parentId = entity.parentId;
335
+ const line = JSON.stringify(entityData);
336
+ // Append to file with fsync for durability (write FIRST, then update cache)
337
+ try {
338
+ const stat = await fs.stat(this.memoryFilePath);
339
+ await this.durableAppendFile(line, stat.size > 0);
340
+ }
341
+ catch (error) {
342
+ // File doesn't exist - create it
343
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
344
+ await this.durableWriteFile(line);
345
+ }
346
+ else {
347
+ throw error;
348
+ }
349
+ }
350
+ // Update cache in-place (after successful file write)
351
+ this.cache.entities.push(entity);
352
+ // Update indexes
353
+ this.nameIndex.add(entity);
354
+ this.typeIndex.add(entity);
355
+ this.lowercaseCache.set(entity);
356
+ this.observationIndex.add(entity.name, entity.observations);
357
+ this.pendingAppends++;
358
+ // Clear search caches
359
+ clearAllSearchCaches();
360
+ // Phase 10 Sprint 2: Emit entity:created event
361
+ this.eventEmitter.emitEntityCreated(entity);
362
+ // Trigger compaction if threshold reached
363
+ if (this.pendingAppends >= this.compactionThreshold) {
364
+ await this.compactInternal();
365
+ }
366
+ });
367
+ }
368
+ /**
369
+ * Append a single relation to the file (O(1) write operation).
370
+ *
371
+ * OPTIMIZED: Uses file append instead of full rewrite.
372
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
373
+ * Updates cache in-place and triggers compaction when threshold is reached.
374
+ *
375
+ * @param relation - The relation to append
376
+ * @returns Promise resolving when append is complete
377
+ */
378
+ async appendRelation(relation) {
379
+ await this.ensureLoaded();
380
+ return this.mutex.runExclusive(async () => {
381
+ const line = JSON.stringify({
382
+ type: 'relation',
383
+ from: relation.from,
384
+ to: relation.to,
385
+ relationType: relation.relationType,
386
+ createdAt: relation.createdAt,
387
+ lastModified: relation.lastModified,
388
+ });
389
+ // Append to file with fsync for durability (write FIRST, then update cache)
390
+ try {
391
+ const stat = await fs.stat(this.memoryFilePath);
392
+ await this.durableAppendFile(line, stat.size > 0);
393
+ }
394
+ catch (error) {
395
+ // File doesn't exist - create it
396
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
397
+ await this.durableWriteFile(line);
398
+ }
399
+ else {
400
+ throw error;
401
+ }
402
+ }
403
+ // Update cache in-place (after successful file write)
404
+ this.cache.relations.push(relation);
405
+ // Update relation index
406
+ this.relationIndex.add(relation);
407
+ this.pendingAppends++;
408
+ // Clear search caches
409
+ clearAllSearchCaches();
410
+ // Phase 10 Sprint 2: Emit relation:created event
411
+ this.eventEmitter.emitRelationCreated(relation);
412
+ // Trigger compaction if threshold reached
413
+ if (this.pendingAppends >= this.compactionThreshold) {
414
+ await this.compactInternal();
415
+ }
416
+ });
417
+ }
418
+ /**
419
+ * Compact the file by rewriting it with only current cache contents.
420
+ *
421
+ * THREAD-SAFE: Uses mutex to prevent concurrent operations.
422
+ * Removes duplicate entries and cleans up the file.
423
+ * Resets pending appends counter.
424
+ *
425
+ * @returns Promise resolving when compaction is complete
426
+ */
427
+ async compact() {
428
+ return this.mutex.runExclusive(async () => {
429
+ await this.compactInternal();
430
+ });
431
+ }
432
+ /**
433
+ * Internal compact implementation (must be called within mutex).
434
+ *
435
+ * @returns Promise resolving when compaction is complete
436
+ */
437
+ async compactInternal() {
438
+ if (this.cache === null) {
439
+ return;
440
+ }
441
+ // Rewrite file with current cache (removes duplicates/updates)
442
+ await this.saveGraphInternal(this.cache);
443
+ this.pendingAppends = 0;
444
+ }
445
+ /**
446
+ * Internal saveGraph implementation (must be called within mutex).
447
+ *
448
+ * @param graph - The knowledge graph to save
449
+ * @returns Promise resolving when save is complete
450
+ */
451
+ async saveGraphInternal(graph) {
120
452
  const lines = [
121
453
  ...graph.entities.map(e => {
122
454
  const entityData = {
@@ -145,11 +477,116 @@ export class GraphStorage {
145
477
  lastModified: r.lastModified,
146
478
  })),
147
479
  ];
148
- await fs.writeFile(this.memoryFilePath, lines.join('\n'));
149
- // Invalidate cache to ensure next load reads fresh data
150
- this.cache = null;
480
+ await this.durableWriteFile(lines.join('\n'));
481
+ // Update cache directly with the saved graph (avoid re-reading from disk)
482
+ this.cache = graph;
483
+ // Rebuild indexes with new graph data
484
+ this.buildEntityIndexes(graph.entities);
485
+ this.buildRelationIndex(graph.relations);
486
+ // Reset pending appends since file is now clean
487
+ this.pendingAppends = 0;
151
488
  // Clear all search caches since graph data has changed
152
489
  clearAllSearchCaches();
490
+ // Phase 10 Sprint 2: Emit graph:saved event
491
+ this.eventEmitter.emitGraphSaved(graph.entities.length, graph.relations.length);
492
+ }
493
+ /**
494
+ * Get the current pending appends count.
495
+ *
496
+ * Useful for testing compaction behavior.
497
+ *
498
+ * @returns Number of pending appends since last compaction
499
+ */
500
+ getPendingAppends() {
501
+ return this.pendingAppends;
502
+ }
503
+ /**
504
+ * Update an entity in-place in the cache and append to file.
505
+ *
506
+ * OPTIMIZED: Modifies cache directly and appends updated version to file.
507
+ * THREAD-SAFE: Uses mutex to prevent concurrent write operations.
508
+ * Does not rewrite the entire file - compaction handles deduplication later.
509
+ *
510
+ * @param entityName - Name of the entity to update
511
+ * @param updates - Partial entity updates to apply
512
+ * @returns Promise resolving to true if entity was found and updated, false otherwise
513
+ */
514
+ async updateEntity(entityName, updates) {
515
+ await this.ensureLoaded();
516
+ return this.mutex.runExclusive(async () => {
517
+ const entityIndex = this.cache.entities.findIndex(e => e.name === entityName);
518
+ if (entityIndex === -1) {
519
+ return false;
520
+ }
521
+ const entity = this.cache.entities[entityIndex];
522
+ const oldType = entity.entityType;
523
+ const timestamp = new Date().toISOString();
524
+ // Phase 10 Sprint 2: Capture previous values for event
525
+ const previousValues = {};
526
+ for (const key of Object.keys(updates)) {
527
+ if (key in entity) {
528
+ previousValues[key] = entity[key];
529
+ }
530
+ }
531
+ // Build the updated entity data for file write BEFORE modifying cache
532
+ // This ensures cache consistency if file write fails
533
+ const updatedEntity = {
534
+ ...entity,
535
+ ...updates,
536
+ lastModified: timestamp,
537
+ };
538
+ const entityData = {
539
+ type: 'entity',
540
+ name: updatedEntity.name,
541
+ entityType: updatedEntity.entityType,
542
+ observations: updatedEntity.observations,
543
+ createdAt: updatedEntity.createdAt,
544
+ lastModified: updatedEntity.lastModified,
545
+ };
546
+ if (updatedEntity.tags !== undefined)
547
+ entityData.tags = updatedEntity.tags;
548
+ if (updatedEntity.importance !== undefined)
549
+ entityData.importance = updatedEntity.importance;
550
+ if (updatedEntity.parentId !== undefined)
551
+ entityData.parentId = updatedEntity.parentId;
552
+ const line = JSON.stringify(entityData);
553
+ // Write to file FIRST with durability - if this fails, cache remains consistent
554
+ try {
555
+ const stat = await fs.stat(this.memoryFilePath);
556
+ await this.durableAppendFile(line, stat.size > 0);
557
+ }
558
+ catch (error) {
559
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
560
+ await this.durableWriteFile(line);
561
+ }
562
+ else {
563
+ throw error;
564
+ }
565
+ }
566
+ // File write succeeded - NOW update cache in-place (sanitized to prevent prototype pollution)
567
+ Object.assign(entity, sanitizeObject(updates));
568
+ entity.lastModified = timestamp;
569
+ // Update indexes
570
+ this.nameIndex.add(entity); // Update reference
571
+ if (updates.entityType && updates.entityType !== oldType) {
572
+ this.typeIndex.updateType(entityName, oldType, updates.entityType);
573
+ }
574
+ this.lowercaseCache.set(entity); // Recompute lowercase
575
+ if (updates.observations) {
576
+ this.observationIndex.remove(entityName); // Remove old observations
577
+ this.observationIndex.add(entityName, entity.observations); // Add new observations
578
+ }
579
+ this.pendingAppends++;
580
+ // Clear search caches
581
+ clearAllSearchCaches();
582
+ // Phase 10 Sprint 2: Emit entity:updated event
583
+ this.eventEmitter.emitEntityUpdated(entityName, updates, previousValues);
584
+ // Trigger compaction if threshold reached
585
+ if (this.pendingAppends >= this.compactionThreshold) {
586
+ await this.compactInternal();
587
+ }
588
+ return true;
589
+ });
153
590
  }
154
591
  /**
155
592
  * Manually clear the cache.
@@ -160,6 +597,7 @@ export class GraphStorage {
160
597
  */
161
598
  clearCache() {
162
599
  this.cache = null;
600
+ this.clearIndexes();
163
601
  }
164
602
  /**
165
603
  * Get the file path being used for storage.
@@ -169,4 +607,179 @@ export class GraphStorage {
169
607
  getFilePath() {
170
608
  return this.memoryFilePath;
171
609
  }
610
+ // ==================== Index Accessors ====================
611
+ /**
612
+ * Get an entity by name in O(1) time.
613
+ *
614
+ * OPTIMIZED: Uses NameIndex for constant-time lookup.
615
+ *
616
+ * @param name - Entity name to look up
617
+ * @returns Entity if found, undefined otherwise
618
+ */
619
+ getEntityByName(name) {
620
+ return this.nameIndex.get(name);
621
+ }
622
+ /**
623
+ * Check if an entity exists by name in O(1) time.
624
+ *
625
+ * @param name - Entity name to check
626
+ * @returns True if entity exists
627
+ */
628
+ hasEntity(name) {
629
+ return this.nameIndex.has(name);
630
+ }
631
+ /**
632
+ * Get all entities of a given type in O(1) time.
633
+ *
634
+ * OPTIMIZED: Uses TypeIndex for constant-time lookup of entity names,
635
+ * then uses NameIndex for O(1) entity retrieval.
636
+ *
637
+ * @param entityType - Entity type to filter by (case-insensitive)
638
+ * @returns Array of entities with the given type
639
+ */
640
+ getEntitiesByType(entityType) {
641
+ const names = this.typeIndex.getNames(entityType);
642
+ const entities = [];
643
+ for (const name of names) {
644
+ const entity = this.nameIndex.get(name);
645
+ if (entity) {
646
+ entities.push(entity);
647
+ }
648
+ }
649
+ return entities;
650
+ }
651
+ /**
652
+ * Get pre-computed lowercase data for an entity.
653
+ *
654
+ * OPTIMIZED: Avoids repeated toLowerCase() calls during search.
655
+ *
656
+ * @param entityName - Entity name to get lowercase data for
657
+ * @returns LowercaseData if entity exists, undefined otherwise
658
+ */
659
+ getLowercased(entityName) {
660
+ return this.lowercaseCache.get(entityName);
661
+ }
662
+ /**
663
+ * Get all unique entity types in the graph.
664
+ *
665
+ * @returns Array of unique entity types (lowercase)
666
+ */
667
+ getEntityTypes() {
668
+ return this.typeIndex.getTypes();
669
+ }
670
+ // ==================== Relation Index Accessors ====================
671
+ /**
672
+ * Get all relations where the entity is the source (outgoing relations) in O(1) time.
673
+ *
674
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
675
+ *
676
+ * @param entityName - Entity name to look up outgoing relations for
677
+ * @returns Array of relations where entity is the source
678
+ */
679
+ getRelationsFrom(entityName) {
680
+ return this.relationIndex.getRelationsFrom(entityName);
681
+ }
682
+ /**
683
+ * Get all relations where the entity is the target (incoming relations) in O(1) time.
684
+ *
685
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
686
+ *
687
+ * @param entityName - Entity name to look up incoming relations for
688
+ * @returns Array of relations where entity is the target
689
+ */
690
+ getRelationsTo(entityName) {
691
+ return this.relationIndex.getRelationsTo(entityName);
692
+ }
693
+ /**
694
+ * Get all relations involving the entity (both incoming and outgoing) in O(1) time.
695
+ *
696
+ * OPTIMIZED: Uses RelationIndex for constant-time lookup.
697
+ *
698
+ * @param entityName - Entity name to look up all relations for
699
+ * @returns Array of all relations involving the entity
700
+ */
701
+ getRelationsFor(entityName) {
702
+ return this.relationIndex.getRelationsFor(entityName);
703
+ }
704
+ /**
705
+ * Check if an entity has any relations.
706
+ *
707
+ * @param entityName - Entity name to check
708
+ * @returns True if entity has any relations
709
+ */
710
+ hasRelations(entityName) {
711
+ return this.relationIndex.hasRelations(entityName);
712
+ }
713
+ // ==================== Observation Index Accessors ====================
714
+ /**
715
+ * Get entities that have observations containing the given word.
716
+ * Uses the observation index for O(1) lookup.
717
+ *
718
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookup instead of
719
+ * linear scan through all entities and their observations.
720
+ *
721
+ * @param word - Word to search for in observations
722
+ * @returns Set of entity names
723
+ */
724
+ getEntitiesByObservationWord(word) {
725
+ return this.observationIndex.getEntitiesWithWord(word);
726
+ }
727
+ /**
728
+ * Get entities that have observations containing ANY of the given words (union).
729
+ * Uses the observation index for O(1) lookup per word.
730
+ *
731
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookups.
732
+ *
733
+ * @param words - Array of words to search for
734
+ * @returns Set of entity names containing any of the words
735
+ */
736
+ getEntitiesByAnyObservationWord(words) {
737
+ return this.observationIndex.getEntitiesWithAnyWord(words);
738
+ }
739
+ /**
740
+ * Get entities that have observations containing ALL of the given words (intersection).
741
+ * Uses the observation index for O(1) lookup per word.
742
+ *
743
+ * OPTIMIZED: Uses ObservationIndex for constant-time lookups and set intersection.
744
+ *
745
+ * @param words - Array of words that must all be present
746
+ * @returns Set of entity names containing all of the words
747
+ */
748
+ getEntitiesByAllObservationWords(words) {
749
+ return this.observationIndex.getEntitiesWithAllWords(words);
750
+ }
751
+ /**
752
+ * Get statistics about the observation index.
753
+ *
754
+ * @returns Object with wordCount and entityCount
755
+ */
756
+ getObservationIndexStats() {
757
+ return this.observationIndex.getStats();
758
+ }
759
+ // ==================== Phase 10 Sprint 1: Transaction Factory ====================
760
+ /**
761
+ * Create a new batch transaction for atomic operations.
762
+ *
763
+ * Returns a BatchTransaction instance that can be used to queue multiple
764
+ * operations and execute them atomically with a single save operation.
765
+ *
766
+ * @returns A new BatchTransaction instance
767
+ *
768
+ * @example
769
+ * ```typescript
770
+ * const storage = new GraphStorage('/data/memory.jsonl');
771
+ *
772
+ * // Create and execute a batch transaction
773
+ * const result = await storage.transaction()
774
+ * .createEntity({ name: 'Alice', entityType: 'person', observations: ['Developer'] })
775
+ * .createEntity({ name: 'Bob', entityType: 'person', observations: ['Designer'] })
776
+ * .createRelation({ from: 'Alice', to: 'Bob', relationType: 'knows' })
777
+ * .execute();
778
+ *
779
+ * console.log(`Batch completed: ${result.operationsExecuted} operations`);
780
+ * ```
781
+ */
782
+ transaction() {
783
+ return new BatchTransaction(this);
784
+ }
172
785
  }