@danielsimonjr/memory-mcp 0.47.1 → 9.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +2000 -194
  3. package/dist/__tests__/file-path.test.js +5 -5
  4. package/dist/__tests__/knowledge-graph.test.js +3 -8
  5. package/dist/core/EntityManager.d.ts +266 -0
  6. package/dist/core/EntityManager.d.ts.map +1 -0
  7. package/dist/core/EntityManager.js +85 -133
  8. package/dist/core/GraphEventEmitter.d.ts +202 -0
  9. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  10. package/dist/core/GraphEventEmitter.js +346 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +643 -31
  14. package/dist/core/GraphTraversal.d.ts +141 -0
  15. package/dist/core/GraphTraversal.d.ts.map +1 -0
  16. package/dist/core/GraphTraversal.js +573 -0
  17. package/dist/core/HierarchyManager.d.ts +111 -0
  18. package/dist/core/HierarchyManager.d.ts.map +1 -0
  19. package/dist/{features → core}/HierarchyManager.js +14 -9
  20. package/dist/core/ManagerContext.d.ts +72 -0
  21. package/dist/core/ManagerContext.d.ts.map +1 -0
  22. package/dist/core/ManagerContext.js +118 -0
  23. package/dist/core/ObservationManager.d.ts +85 -0
  24. package/dist/core/ObservationManager.d.ts.map +1 -0
  25. package/dist/core/ObservationManager.js +51 -57
  26. package/dist/core/RelationManager.d.ts +131 -0
  27. package/dist/core/RelationManager.d.ts.map +1 -0
  28. package/dist/core/RelationManager.js +31 -7
  29. package/dist/core/SQLiteStorage.d.ts +354 -0
  30. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  31. package/dist/core/SQLiteStorage.js +917 -0
  32. package/dist/core/StorageFactory.d.ts +45 -0
  33. package/dist/core/StorageFactory.d.ts.map +1 -0
  34. package/dist/core/StorageFactory.js +64 -0
  35. package/dist/core/TransactionManager.d.ts +464 -0
  36. package/dist/core/TransactionManager.d.ts.map +1 -0
  37. package/dist/core/TransactionManager.js +490 -13
  38. package/dist/core/index.d.ts +17 -0
  39. package/dist/core/index.d.ts.map +1 -0
  40. package/dist/core/index.js +12 -2
  41. package/dist/features/AnalyticsManager.d.ts +44 -0
  42. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  43. package/dist/features/AnalyticsManager.js +14 -13
  44. package/dist/features/ArchiveManager.d.ts +133 -0
  45. package/dist/features/ArchiveManager.d.ts.map +1 -0
  46. package/dist/features/ArchiveManager.js +221 -14
  47. package/dist/features/CompressionManager.d.ts +117 -0
  48. package/dist/features/CompressionManager.d.ts.map +1 -0
  49. package/dist/features/CompressionManager.js +189 -20
  50. package/dist/features/IOManager.d.ts +225 -0
  51. package/dist/features/IOManager.d.ts.map +1 -0
  52. package/dist/features/IOManager.js +1041 -0
  53. package/dist/features/StreamingExporter.d.ts +123 -0
  54. package/dist/features/StreamingExporter.d.ts.map +1 -0
  55. package/dist/features/StreamingExporter.js +203 -0
  56. package/dist/features/TagManager.d.ts +147 -0
  57. package/dist/features/TagManager.d.ts.map +1 -0
  58. package/dist/features/index.d.ts +12 -0
  59. package/dist/features/index.d.ts.map +1 -0
  60. package/dist/features/index.js +5 -6
  61. package/dist/index.d.ts +9 -0
  62. package/dist/index.d.ts.map +1 -0
  63. package/dist/index.js +12 -45
  64. package/dist/memory.jsonl +1 -18
  65. package/dist/search/BasicSearch.d.ts +51 -0
  66. package/dist/search/BasicSearch.d.ts.map +1 -0
  67. package/dist/search/BasicSearch.js +9 -3
  68. package/dist/search/BooleanSearch.d.ts +98 -0
  69. package/dist/search/BooleanSearch.d.ts.map +1 -0
  70. package/dist/search/BooleanSearch.js +156 -9
  71. package/dist/search/EmbeddingService.d.ts +178 -0
  72. package/dist/search/EmbeddingService.d.ts.map +1 -0
  73. package/dist/search/EmbeddingService.js +358 -0
  74. package/dist/search/FuzzySearch.d.ts +118 -0
  75. package/dist/search/FuzzySearch.d.ts.map +1 -0
  76. package/dist/search/FuzzySearch.js +241 -25
  77. package/dist/search/QueryCostEstimator.d.ts +111 -0
  78. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  79. package/dist/search/QueryCostEstimator.js +355 -0
  80. package/dist/search/RankedSearch.d.ts +71 -0
  81. package/dist/search/RankedSearch.d.ts.map +1 -0
  82. package/dist/search/RankedSearch.js +54 -6
  83. package/dist/search/SavedSearchManager.d.ts +79 -0
  84. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  85. package/dist/search/SearchFilterChain.d.ts +120 -0
  86. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  87. package/dist/search/SearchFilterChain.js +2 -4
  88. package/dist/search/SearchManager.d.ts +326 -0
  89. package/dist/search/SearchManager.d.ts.map +1 -0
  90. package/dist/search/SearchManager.js +148 -0
  91. package/dist/search/SearchSuggestions.d.ts +27 -0
  92. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  93. package/dist/search/SearchSuggestions.js +1 -1
  94. package/dist/search/SemanticSearch.d.ts +149 -0
  95. package/dist/search/SemanticSearch.d.ts.map +1 -0
  96. package/dist/search/SemanticSearch.js +323 -0
  97. package/dist/search/TFIDFEventSync.d.ts +85 -0
  98. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  99. package/dist/search/TFIDFEventSync.js +133 -0
  100. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  101. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  102. package/dist/search/TFIDFIndexManager.js +232 -17
  103. package/dist/search/VectorStore.d.ts +235 -0
  104. package/dist/search/VectorStore.d.ts.map +1 -0
  105. package/dist/search/VectorStore.js +311 -0
  106. package/dist/search/index.d.ts +21 -0
  107. package/dist/search/index.d.ts.map +1 -0
  108. package/dist/search/index.js +12 -0
  109. package/dist/server/MCPServer.d.ts +21 -0
  110. package/dist/server/MCPServer.d.ts.map +1 -0
  111. package/dist/server/MCPServer.js +4 -4
  112. package/dist/server/responseCompressor.d.ts +94 -0
  113. package/dist/server/responseCompressor.d.ts.map +1 -0
  114. package/dist/server/responseCompressor.js +127 -0
  115. package/dist/server/toolDefinitions.d.ts +27 -0
  116. package/dist/server/toolDefinitions.d.ts.map +1 -0
  117. package/dist/server/toolDefinitions.js +189 -18
  118. package/dist/server/toolHandlers.d.ts +41 -0
  119. package/dist/server/toolHandlers.d.ts.map +1 -0
  120. package/dist/server/toolHandlers.js +467 -75
  121. package/dist/types/index.d.ts +13 -0
  122. package/dist/types/index.d.ts.map +1 -0
  123. package/dist/types/index.js +1 -1
  124. package/dist/types/types.d.ts +1654 -0
  125. package/dist/types/types.d.ts.map +1 -0
  126. package/dist/types/types.js +9 -0
  127. package/dist/utils/compressedCache.d.ts +192 -0
  128. package/dist/utils/compressedCache.d.ts.map +1 -0
  129. package/dist/utils/compressedCache.js +309 -0
  130. package/dist/utils/compressionUtil.d.ts +214 -0
  131. package/dist/utils/compressionUtil.d.ts.map +1 -0
  132. package/dist/utils/compressionUtil.js +247 -0
  133. package/dist/utils/constants.d.ts +245 -0
  134. package/dist/utils/constants.d.ts.map +1 -0
  135. package/dist/utils/constants.js +124 -0
  136. package/dist/utils/entityUtils.d.ts +321 -0
  137. package/dist/utils/entityUtils.d.ts.map +1 -0
  138. package/dist/utils/entityUtils.js +434 -4
  139. package/dist/utils/errors.d.ts +95 -0
  140. package/dist/utils/errors.d.ts.map +1 -0
  141. package/dist/utils/errors.js +24 -0
  142. package/dist/utils/formatters.d.ts +145 -0
  143. package/dist/utils/formatters.d.ts.map +1 -0
  144. package/dist/utils/{paginationUtils.js → formatters.js} +54 -3
  145. package/dist/utils/index.d.ts +23 -0
  146. package/dist/utils/index.d.ts.map +1 -0
  147. package/dist/utils/index.js +69 -31
  148. package/dist/utils/indexes.d.ts +270 -0
  149. package/dist/utils/indexes.d.ts.map +1 -0
  150. package/dist/utils/indexes.js +526 -0
  151. package/dist/utils/logger.d.ts +24 -0
  152. package/dist/utils/logger.d.ts.map +1 -0
  153. package/dist/utils/operationUtils.d.ts +124 -0
  154. package/dist/utils/operationUtils.d.ts.map +1 -0
  155. package/dist/utils/operationUtils.js +175 -0
  156. package/dist/utils/parallelUtils.d.ts +72 -0
  157. package/dist/utils/parallelUtils.d.ts.map +1 -0
  158. package/dist/utils/parallelUtils.js +169 -0
  159. package/dist/utils/schemas.d.ts +374 -0
  160. package/dist/utils/schemas.d.ts.map +1 -0
  161. package/dist/utils/schemas.js +302 -2
  162. package/dist/utils/searchAlgorithms.d.ts +99 -0
  163. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  164. package/dist/utils/searchAlgorithms.js +167 -0
  165. package/dist/utils/searchCache.d.ts +108 -0
  166. package/dist/utils/searchCache.d.ts.map +1 -0
  167. package/dist/utils/taskScheduler.d.ts +290 -0
  168. package/dist/utils/taskScheduler.d.ts.map +1 -0
  169. package/dist/utils/taskScheduler.js +466 -0
  170. package/dist/workers/index.d.ts +12 -0
  171. package/dist/workers/index.d.ts.map +1 -0
  172. package/dist/workers/index.js +9 -0
  173. package/dist/workers/levenshteinWorker.d.ts +60 -0
  174. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  175. package/dist/workers/levenshteinWorker.js +98 -0
  176. package/package.json +17 -4
  177. package/dist/__tests__/edge-cases/edge-cases.test.js +0 -406
  178. package/dist/__tests__/integration/workflows.test.js +0 -449
  179. package/dist/__tests__/performance/benchmarks.test.js +0 -413
  180. package/dist/__tests__/unit/core/EntityManager.test.js +0 -334
  181. package/dist/__tests__/unit/core/GraphStorage.test.js +0 -205
  182. package/dist/__tests__/unit/core/RelationManager.test.js +0 -274
  183. package/dist/__tests__/unit/features/CompressionManager.test.js +0 -350
  184. package/dist/__tests__/unit/search/BasicSearch.test.js +0 -311
  185. package/dist/__tests__/unit/search/BooleanSearch.test.js +0 -432
  186. package/dist/__tests__/unit/search/FuzzySearch.test.js +0 -448
  187. package/dist/__tests__/unit/search/RankedSearch.test.js +0 -379
  188. package/dist/__tests__/unit/utils/levenshtein.test.js +0 -77
  189. package/dist/core/KnowledgeGraphManager.js +0 -423
  190. package/dist/features/BackupManager.js +0 -311
  191. package/dist/features/ExportManager.js +0 -305
  192. package/dist/features/ImportExportManager.js +0 -50
  193. package/dist/features/ImportManager.js +0 -328
  194. package/dist/types/analytics.types.js +0 -6
  195. package/dist/types/entity.types.js +0 -7
  196. package/dist/types/import-export.types.js +0 -7
  197. package/dist/types/search.types.js +0 -7
  198. package/dist/types/tag.types.js +0 -6
  199. package/dist/utils/dateUtils.js +0 -89
  200. package/dist/utils/filterUtils.js +0 -155
  201. package/dist/utils/levenshtein.js +0 -62
  202. package/dist/utils/pathUtils.js +0 -115
  203. package/dist/utils/responseFormatter.js +0 -55
  204. package/dist/utils/tagUtils.js +0 -107
  205. package/dist/utils/tfidf.js +0 -90
  206. package/dist/utils/validationHelper.js +0 -99
  207. package/dist/utils/validationUtils.js +0 -109
@@ -1,27 +1,60 @@
1
1
  /**
2
2
  * Compression Manager
3
3
  *
4
- * Detects and merges duplicate entities to compress the knowledge graph.
4
+ * Handles duplicate detection, entity merging, and graph compression.
5
+ * Extracted from SearchManager (Phase 4: Consolidate God Objects).
5
6
  *
6
7
  * @module features/CompressionManager
7
8
  */
8
- import { levenshteinDistance } from '../utils/levenshtein.js';
9
+ import { levenshteinDistance, checkCancellation, createProgressReporter, createProgress, } from '../utils/index.js';
9
10
  import { EntityNotFoundError, InsufficientEntitiesError } from '../utils/errors.js';
10
11
  import { SIMILARITY_WEIGHTS, DEFAULT_DUPLICATE_THRESHOLD } from '../utils/constants.js';
11
12
  /**
12
- * Manages graph compression through duplicate detection and merging.
13
+ * Manages compression operations for the knowledge graph.
13
14
  */
14
15
  export class CompressionManager {
15
16
  storage;
16
17
  constructor(storage) {
17
18
  this.storage = storage;
18
19
  }
20
+ /**
21
+ * Prepare an entity for efficient similarity comparisons.
22
+ * Pre-computes all normalized data to avoid repeated computation.
23
+ *
24
+ * @param entity - The entity to prepare
25
+ * @returns PreparedEntity with pre-computed data
26
+ */
27
+ prepareEntity(entity) {
28
+ return {
29
+ entity,
30
+ nameLower: entity.name.toLowerCase(),
31
+ typeLower: entity.entityType.toLowerCase(),
32
+ observationSet: new Set(entity.observations.map(o => o.toLowerCase())),
33
+ tagSet: new Set((entity.tags ?? []).map(t => t.toLowerCase())),
34
+ };
35
+ }
36
+ /**
37
+ * Prepare multiple entities for efficient similarity comparisons.
38
+ * Use this before batch comparison operations.
39
+ *
40
+ * @param entities - Entities to prepare
41
+ * @returns Map of entity name to PreparedEntity
42
+ */
43
+ prepareEntities(entities) {
44
+ const prepared = new Map();
45
+ for (const entity of entities) {
46
+ prepared.set(entity.name, this.prepareEntity(entity));
47
+ }
48
+ return prepared;
49
+ }
19
50
  /**
20
51
  * Calculate similarity between two entities using multiple heuristics.
21
52
  *
22
53
  * Uses configurable weights defined in SIMILARITY_WEIGHTS constant.
23
54
  * See SIMILARITY_WEIGHTS for the breakdown of scoring factors.
24
55
  *
56
+ * NOTE: For batch comparisons, use prepareEntities() + calculatePreparedSimilarity() for better performance.
57
+ *
25
58
  * @param e1 - First entity
26
59
  * @param e2 - Second entity
27
60
  * @returns Similarity score from 0 (completely different) to 1 (identical)
@@ -60,6 +93,58 @@ export class CompressionManager {
60
93
  }
61
94
  return factors > 0 ? score / factors : 0;
62
95
  }
96
+ /**
97
+ * Efficiently calculate intersection size of two Sets without creating a new Set.
98
+ * Iterates over the smaller set for O(min(m,n)) complexity.
99
+ */
100
+ setIntersectionSize(a, b) {
101
+ // Always iterate over smaller set
102
+ const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a];
103
+ let count = 0;
104
+ for (const item of smaller) {
105
+ if (larger.has(item))
106
+ count++;
107
+ }
108
+ return count;
109
+ }
110
+ /**
111
+ * Calculate similarity between two prepared entities.
112
+ * OPTIMIZED: Uses pre-computed Sets to avoid O(n) set creation per comparison.
113
+ *
114
+ * @param p1 - First prepared entity
115
+ * @param p2 - Second prepared entity
116
+ * @returns Similarity score from 0 (completely different) to 1 (identical)
117
+ */
118
+ calculatePreparedSimilarity(p1, p2) {
119
+ let score = 0;
120
+ let factors = 0;
121
+ // Name similarity (Levenshtein-based) - use pre-computed lowercase
122
+ const nameDistance = levenshteinDistance(p1.nameLower, p2.nameLower);
123
+ const maxNameLength = Math.max(p1.nameLower.length, p2.nameLower.length);
124
+ const nameSimilarity = 1 - nameDistance / maxNameLength;
125
+ score += nameSimilarity * SIMILARITY_WEIGHTS.NAME;
126
+ factors += SIMILARITY_WEIGHTS.NAME;
127
+ // Type similarity (exact match) - use pre-computed lowercase
128
+ if (p1.typeLower === p2.typeLower) {
129
+ score += SIMILARITY_WEIGHTS.TYPE;
130
+ }
131
+ factors += SIMILARITY_WEIGHTS.TYPE;
132
+ // Observation overlap (Jaccard similarity) - use pre-computed Sets
133
+ const obsIntersectionSize = this.setIntersectionSize(p1.observationSet, p2.observationSet);
134
+ const obsUnionSize = p1.observationSet.size + p2.observationSet.size - obsIntersectionSize;
135
+ const observationSimilarity = obsUnionSize > 0 ? obsIntersectionSize / obsUnionSize : 0;
136
+ score += observationSimilarity * SIMILARITY_WEIGHTS.OBSERVATIONS;
137
+ factors += SIMILARITY_WEIGHTS.OBSERVATIONS;
138
+ // Tag overlap (Jaccard similarity) - use pre-computed Sets
139
+ if (p1.tagSet.size > 0 || p2.tagSet.size > 0) {
140
+ const tagIntersectionSize = this.setIntersectionSize(p1.tagSet, p2.tagSet);
141
+ const tagUnionSize = p1.tagSet.size + p2.tagSet.size - tagIntersectionSize;
142
+ const tagSimilarity = tagUnionSize > 0 ? tagIntersectionSize / tagUnionSize : 0;
143
+ score += tagSimilarity * SIMILARITY_WEIGHTS.TAGS;
144
+ factors += SIMILARITY_WEIGHTS.TAGS;
145
+ }
146
+ return factors > 0 ? score / factors : 0;
147
+ }
63
148
  /**
64
149
  * Find duplicate entities in the graph based on similarity threshold.
65
150
  *
@@ -68,15 +153,28 @@ export class CompressionManager {
68
153
  * 2. Within each type, buckets by name prefix (first 2 chars normalized)
69
154
  * 3. Only compares entities within same or adjacent buckets
70
155
  *
156
+ * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
157
+ *
71
158
  * Complexity: O(n·k) where k is average bucket size (typically << n)
72
159
  *
73
160
  * @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
161
+ * @param options - Optional progress/cancellation options (Phase 9B)
74
162
  * @returns Array of duplicate groups (each group has similar entities)
163
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
75
164
  */
76
- async findDuplicates(threshold = DEFAULT_DUPLICATE_THRESHOLD) {
165
+ async findDuplicates(threshold = DEFAULT_DUPLICATE_THRESHOLD, options) {
166
+ // Check for early cancellation
167
+ checkCancellation(options?.signal, 'findDuplicates');
77
168
  const graph = await this.storage.loadGraph();
78
169
  const duplicateGroups = [];
79
170
  const processed = new Set();
171
+ // Setup progress reporter
172
+ const reportProgress = createProgressReporter(options?.onProgress);
173
+ const totalEntities = graph.entities.length;
174
+ let processedCount = 0;
175
+ reportProgress?.(createProgress(0, totalEntities, 'findDuplicates'));
176
+ // OPTIMIZATION: Pre-prepare all entities once before comparisons
177
+ const preparedEntities = this.prepareEntities(graph.entities);
80
178
  // Step 1: Bucket entities by type (reduces comparisons drastically)
81
179
  const typeMap = new Map();
82
180
  for (const entity of graph.entities) {
@@ -88,9 +186,14 @@ export class CompressionManager {
88
186
  }
89
187
  // Step 2: For each type bucket, sub-bucket by name prefix
90
188
  for (const entities of typeMap.values()) {
189
+ // Check for cancellation between type buckets
190
+ checkCancellation(options?.signal, 'findDuplicates');
91
191
  // Skip single-entity types (no duplicates possible)
92
- if (entities.length < 2)
192
+ if (entities.length < 2) {
193
+ processedCount += entities.length;
194
+ reportProgress?.(createProgress(processedCount, totalEntities, 'findDuplicates'));
93
195
  continue;
196
+ }
94
197
  // Create name prefix buckets (first 2 chars, normalized)
95
198
  const prefixMap = new Map();
96
199
  for (const entity of entities) {
@@ -103,6 +206,8 @@ export class CompressionManager {
103
206
  // Step 3: Compare only within buckets (or adjacent buckets for fuzzy matching)
104
207
  const prefixKeys = Array.from(prefixMap.keys()).sort();
105
208
  for (let bucketIdx = 0; bucketIdx < prefixKeys.length; bucketIdx++) {
209
+ // Check for cancellation between prefix buckets
210
+ checkCancellation(options?.signal, 'findDuplicates');
106
211
  const currentPrefix = prefixKeys[bucketIdx];
107
212
  const currentBucket = prefixMap.get(currentPrefix);
108
213
  // Collect entities to compare: current bucket + adjacent buckets
@@ -116,12 +221,16 @@ export class CompressionManager {
116
221
  const entity1 = currentBucket[i];
117
222
  if (processed.has(entity1.name))
118
223
  continue;
224
+ // OPTIMIZATION: Use prepared entity for comparison
225
+ const prepared1 = preparedEntities.get(entity1.name);
119
226
  const group = [entity1.name];
120
227
  for (let j = 0; j < candidateEntities.length; j++) {
121
228
  const entity2 = candidateEntities[j];
122
229
  if (entity1.name === entity2.name || processed.has(entity2.name))
123
230
  continue;
124
- const similarity = this.calculateEntitySimilarity(entity1, entity2);
231
+ // OPTIMIZATION: Use prepared entity and optimized similarity
232
+ const prepared2 = preparedEntities.get(entity2.name);
233
+ const similarity = this.calculatePreparedSimilarity(prepared1, prepared2);
125
234
  if (similarity >= threshold) {
126
235
  group.push(entity2.name);
127
236
  processed.add(entity2.name);
@@ -131,9 +240,13 @@ export class CompressionManager {
131
240
  duplicateGroups.push(group);
132
241
  processed.add(entity1.name);
133
242
  }
243
+ processedCount++;
244
+ reportProgress?.(createProgress(processedCount, totalEntities, 'findDuplicates'));
134
245
  }
135
246
  }
136
247
  }
248
+ // Report completion
249
+ reportProgress?.(createProgress(totalEntities, totalEntities, 'findDuplicates'));
137
250
  return duplicateGroups;
138
251
  }
139
252
  /**
@@ -150,15 +263,19 @@ export class CompressionManager {
150
263
  *
151
264
  * @param entityNames - Names of entities to merge (first one is kept)
152
265
  * @param targetName - Optional new name for merged entity (default: first entity name)
266
+ * @param options - Optional configuration
267
+ * @param options.graph - Pre-loaded graph to use (avoids reload)
268
+ * @param options.skipSave - If true, don't save (caller will save)
153
269
  * @returns The merged entity
154
270
  * @throws {InsufficientEntitiesError} If less than 2 entities provided
155
271
  * @throws {EntityNotFoundError} If any entity not found
156
272
  */
157
- async mergeEntities(entityNames, targetName) {
273
+ async mergeEntities(entityNames, targetName, options = {}) {
158
274
  if (entityNames.length < 2) {
159
275
  throw new InsufficientEntitiesError('merging', 2, entityNames.length);
160
276
  }
161
- const graph = await this.storage.loadGraph();
277
+ // Use provided graph or load fresh
278
+ const graph = options.graph ?? await this.storage.getGraphForMutation();
162
279
  const entitiesToMerge = entityNames.map(name => {
163
280
  const entity = graph.entities.find(e => e.name === name);
164
281
  if (!entity) {
@@ -232,20 +349,45 @@ export class CompressionManager {
232
349
  // Remove merged entities
233
350
  const mergeNames = new Set(mergeEntities.map(e => e.name));
234
351
  graph.entities = graph.entities.filter(e => !mergeNames.has(e.name));
235
- await this.storage.saveGraph(graph);
352
+ // Save unless caller said to skip
353
+ if (!options.skipSave) {
354
+ await this.storage.saveGraph(graph);
355
+ }
236
356
  return keepEntity;
237
357
  }
238
358
  /**
239
359
  * Compress the knowledge graph by finding and merging duplicates.
360
+ * OPTIMIZED: Loads graph once, performs all merges, saves once.
361
+ *
362
+ * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
240
363
  *
241
364
  * @param threshold - Similarity threshold for duplicate detection (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
242
365
  * @param dryRun - If true, only report what would be compressed without applying changes
366
+ * @param options - Optional progress/cancellation options (Phase 9B)
243
367
  * @returns Compression result with statistics
368
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
244
369
  */
245
- async compressGraph(threshold = DEFAULT_DUPLICATE_THRESHOLD, dryRun = false) {
246
- const initialGraph = await this.storage.loadGraph();
247
- const initialSize = JSON.stringify(initialGraph).length;
248
- const duplicateGroups = await this.findDuplicates(threshold);
370
+ async compressGraph(threshold = DEFAULT_DUPLICATE_THRESHOLD, dryRun = false, options) {
371
+ // Check for early cancellation
372
+ checkCancellation(options?.signal, 'compressGraph');
373
+ // Setup progress reporter (we'll use phases: 50% finding duplicates, 50% merging)
374
+ const reportProgress = createProgressReporter(options?.onProgress);
375
+ reportProgress?.(createProgress(0, 100, 'compressGraph'));
376
+ // Phase 1: Find duplicates (0-50% progress)
377
+ const duplicateGroups = await this.findDuplicates(threshold, {
378
+ signal: options?.signal,
379
+ onProgress: (p) => {
380
+ // Map findDuplicates progress (0-100%) to compressGraph progress (0-50%)
381
+ const compressProgress = Math.round(p.percentage * 0.5);
382
+ reportProgress?.(createProgress(compressProgress, 100, 'finding duplicates'));
383
+ },
384
+ });
385
+ // Check for cancellation after finding duplicates
386
+ checkCancellation(options?.signal, 'compressGraph');
387
+ reportProgress?.(createProgress(50, 100, 'compressGraph'));
388
+ // OPTIMIZATION: Load graph once for all operations
389
+ const graph = await this.storage.getGraphForMutation();
390
+ const initialSize = JSON.stringify(graph).length;
249
391
  const result = {
250
392
  duplicatesFound: duplicateGroups.reduce((sum, group) => sum + group.length, 0),
251
393
  entitiesMerged: 0,
@@ -263,12 +405,32 @@ export class CompressionManager {
263
405
  });
264
406
  result.entitiesMerged += group.length - 1;
265
407
  }
408
+ reportProgress?.(createProgress(100, 100, 'compressGraph'));
266
409
  return result;
267
410
  }
268
- // Actually merge duplicates
411
+ // Phase 2: Merge duplicates (50-100% progress)
412
+ const totalGroups = duplicateGroups.length;
413
+ let mergedGroups = 0;
414
+ // Merge all duplicates using the same graph instance
269
415
  for (const group of duplicateGroups) {
416
+ // Check for cancellation between merges
417
+ checkCancellation(options?.signal, 'compressGraph');
270
418
  try {
271
- await this.mergeEntities(group);
419
+ // Count observations before merge using loaded graph
420
+ let totalObservationsBefore = 0;
421
+ for (const name of group) {
422
+ const entity = graph.entities.find(e => e.name === name);
423
+ if (entity) {
424
+ totalObservationsBefore += entity.observations.length;
425
+ }
426
+ }
427
+ // OPTIMIZATION: Pass graph and skip individual saves
428
+ const mergedEntity = await this.mergeEntities(group, undefined, {
429
+ graph,
430
+ skipSave: true,
431
+ });
432
+ const observationsAfter = mergedEntity.observations.length;
433
+ result.observationsCompressed += totalObservationsBefore - observationsAfter;
272
434
  result.mergedEntities.push({
273
435
  kept: group[0],
274
436
  merged: group.slice(1),
@@ -279,13 +441,20 @@ export class CompressionManager {
279
441
  // Skip groups that fail to merge
280
442
  console.error(`Failed to merge group ${group}:`, error);
281
443
  }
444
+ mergedGroups++;
445
+ // Map merge progress (0-100%) to compressGraph progress (50-100%)
446
+ const mergeProgress = totalGroups > 0 ? Math.round(50 + (mergedGroups / totalGroups) * 50) : 100;
447
+ reportProgress?.(createProgress(mergeProgress, 100, 'merging entities'));
282
448
  }
283
- // Calculate space saved
284
- const finalGraph = await this.storage.loadGraph();
285
- const finalSize = JSON.stringify(finalGraph).length;
449
+ // Check for cancellation before final save
450
+ checkCancellation(options?.signal, 'compressGraph');
451
+ // OPTIMIZATION: Save once after all merges complete
452
+ await this.storage.saveGraph(graph);
453
+ const finalSize = JSON.stringify(graph).length;
286
454
  result.spaceFreed = initialSize - finalSize;
287
- // Count compressed observations (approximation)
288
- result.observationsCompressed = result.entitiesMerged;
455
+ result.relationsConsolidated = result.entitiesMerged;
456
+ // Report completion
457
+ reportProgress?.(createProgress(100, 100, 'compressGraph'));
289
458
  return result;
290
459
  }
291
460
  }
@@ -0,0 +1,225 @@
1
+ /**
2
+ * IO Manager
3
+ *
4
+ * Unified manager for import, export, and backup operations.
5
+ * Consolidates BackupManager, ExportManager, and ImportManager (Sprint 11.4).
6
+ *
7
+ * @module features/IOManager
8
+ */
9
+ import type { ReadonlyKnowledgeGraph, ImportResult, BackupOptions, BackupResult, RestoreResult, ExportOptions, ExportResult, LongRunningOperationOptions } from '../types/index.js';
10
+ import type { GraphStorage } from '../core/GraphStorage.js';
11
+ /**
12
+ * Supported export formats.
13
+ */
14
+ export type ExportFormat = 'json' | 'csv' | 'graphml' | 'gexf' | 'dot' | 'markdown' | 'mermaid';
15
+ /**
16
+ * Supported import formats.
17
+ */
18
+ export type ImportFormat = 'json' | 'csv' | 'graphml';
19
+ /**
20
+ * Merge strategies for handling existing entities during import.
21
+ */
22
+ export type MergeStrategy = 'replace' | 'skip' | 'merge' | 'fail';
23
+ /**
24
+ * Metadata stored with each backup.
25
+ * Extended with compression information for Phase 3 Sprint 2.
26
+ */
27
+ export interface BackupMetadata {
28
+ /** Timestamp when backup was created (ISO 8601) */
29
+ timestamp: string;
30
+ /** Number of entities in the backup */
31
+ entityCount: number;
32
+ /** Number of relations in the backup */
33
+ relationCount: number;
34
+ /** File size in bytes (compressed size if compressed) */
35
+ fileSize: number;
36
+ /** Optional description/reason for backup */
37
+ description?: string;
38
+ /** Whether the backup is compressed (default: true for new backups) */
39
+ compressed?: boolean;
40
+ /** Original size before compression in bytes */
41
+ originalSize?: number;
42
+ /** Compression ratio achieved (compressedSize / originalSize) */
43
+ compressionRatio?: number;
44
+ /** Compression format used */
45
+ compressionFormat?: 'brotli' | 'none';
46
+ }
47
+ /**
48
+ * Information about a backup file.
49
+ * Extended with compression details for Phase 3 Sprint 2.
50
+ */
51
+ export interface BackupInfo {
52
+ /** Backup file name */
53
+ fileName: string;
54
+ /** Full path to backup file */
55
+ filePath: string;
56
+ /** Backup metadata */
57
+ metadata: BackupMetadata;
58
+ /** Whether the backup is compressed */
59
+ compressed: boolean;
60
+ /** File size in bytes */
61
+ size: number;
62
+ }
63
+ /**
64
+ * Unified manager for import, export, and backup operations.
65
+ *
66
+ * Combines functionality from:
67
+ * - ExportManager: Graph export to various formats
68
+ * - ImportManager: Graph import from various formats
69
+ * - BackupManager: Point-in-time backup and restore
70
+ */
71
+ export declare class IOManager {
72
+ private storage;
73
+ private readonly backupDir;
74
+ constructor(storage: GraphStorage);
75
+ /**
76
+ * Export graph to specified format.
77
+ *
78
+ * @param graph - Knowledge graph to export
79
+ * @param format - Export format
80
+ * @returns Formatted export string
81
+ */
82
+ exportGraph(graph: ReadonlyKnowledgeGraph, format: ExportFormat): string;
83
+ /**
84
+ * Export graph with optional brotli compression.
85
+ *
86
+ * Compression is applied when:
87
+ * - `options.compress` is explicitly set to `true`
88
+ * - The exported content exceeds 100KB (auto-compress threshold)
89
+ *
90
+ * Compressed content is returned as base64-encoded string.
91
+ * Uncompressed content is returned as UTF-8 string.
92
+ *
93
+ * @param graph - Knowledge graph to export
94
+ * @param format - Export format
95
+ * @param options - Export options including compression settings
96
+ * @returns Export result with content and compression metadata
97
+ *
98
+ * @example
99
+ * ```typescript
100
+ * // Export with explicit compression
101
+ * const result = await manager.exportGraphWithCompression(graph, 'json', {
102
+ * compress: true,
103
+ * compressionQuality: 11
104
+ * });
105
+ *
106
+ * // Export with auto-compression for large graphs
107
+ * const result = await manager.exportGraphWithCompression(graph, 'json');
108
+ * // Compresses automatically if content > 100KB
109
+ * ```
110
+ */
111
+ exportGraphWithCompression(graph: ReadonlyKnowledgeGraph, format: ExportFormat, options?: ExportOptions): Promise<ExportResult>;
112
+ /**
113
+ * Stream export to a file for large graphs.
114
+ *
115
+ * Uses StreamingExporter to write entities and relations incrementally
116
+ * to avoid loading the entire export content into memory.
117
+ *
118
+ * @param format - Export format
119
+ * @param graph - Knowledge graph to export
120
+ * @param options - Export options with required outputPath
121
+ * @returns Export result with streaming metadata
122
+ * @private
123
+ */
124
+ private streamExport;
125
+ private exportAsJson;
126
+ private exportAsCsv;
127
+ private exportAsGraphML;
128
+ private exportAsGEXF;
129
+ private exportAsDOT;
130
+ private exportAsMarkdown;
131
+ private exportAsMermaid;
132
+ /**
133
+ * Import graph from formatted data.
134
+ *
135
+ * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
136
+ *
137
+ * @param format - Import format
138
+ * @param data - Import data string
139
+ * @param mergeStrategy - How to handle conflicts
140
+ * @param dryRun - If true, preview changes without applying
141
+ * @param options - Optional progress/cancellation options (Phase 9B)
142
+ * @returns Import result with statistics
143
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
144
+ */
145
+ importGraph(format: ImportFormat, data: string, mergeStrategy?: MergeStrategy, dryRun?: boolean, options?: LongRunningOperationOptions): Promise<ImportResult>;
146
+ private parseJsonImport;
147
+ private parseCsvImport;
148
+ private parseGraphMLImport;
149
+ private mergeImportedGraph;
150
+ /**
151
+ * Ensure backup directory exists.
152
+ */
153
+ private ensureBackupDir;
154
+ /**
155
+ * Generate backup file name with timestamp.
156
+ * @param compressed - Whether the backup will be compressed (affects extension)
157
+ */
158
+ private generateBackupFileName;
159
+ /**
160
+ * Create a backup of the current knowledge graph.
161
+ *
162
+ * By default, backups are compressed with brotli for 50-70% space reduction.
163
+ * Use `options.compress = false` to create uncompressed backups.
164
+ *
165
+ * @param options - Backup options (compress, description) or legacy description string
166
+ * @returns Promise resolving to BackupResult with compression statistics
167
+ *
168
+ * @example
169
+ * ```typescript
170
+ * // Compressed backup (default)
171
+ * const result = await manager.createBackup({ description: 'Pre-migration backup' });
172
+ * console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
173
+ *
174
+ * // Uncompressed backup
175
+ * const result = await manager.createBackup({ compress: false });
176
+ * ```
177
+ */
178
+ createBackup(options?: BackupOptions | string): Promise<BackupResult>;
179
+ /**
180
+ * List all available backups, sorted by timestamp (newest first).
181
+ *
182
+ * Detects both compressed (.jsonl.br) and uncompressed (.jsonl) backups.
183
+ *
184
+ * @returns Promise resolving to array of backup information with compression details
185
+ */
186
+ listBackups(): Promise<BackupInfo[]>;
187
+ /**
188
+ * Restore the knowledge graph from a backup file.
189
+ *
190
+ * Automatically detects and decompresses brotli-compressed backups (.br extension).
191
+ * Maintains backward compatibility with uncompressed backups.
192
+ *
193
+ * @param backupPath - Path to the backup file to restore from
194
+ * @returns Promise resolving to RestoreResult with restoration details
195
+ *
196
+ * @example
197
+ * ```typescript
198
+ * // Restore from compressed backup
199
+ * const result = await manager.restoreFromBackup('/path/to/backup.jsonl.br');
200
+ * console.log(`Restored ${result.entityCount} entities from compressed backup`);
201
+ *
202
+ * // Restore from uncompressed backup (legacy)
203
+ * const result = await manager.restoreFromBackup('/path/to/backup.jsonl');
204
+ * ```
205
+ */
206
+ restoreFromBackup(backupPath: string): Promise<RestoreResult>;
207
+ /**
208
+ * Delete a specific backup file.
209
+ *
210
+ * @param backupPath - Path to the backup file to delete
211
+ */
212
+ deleteBackup(backupPath: string): Promise<void>;
213
+ /**
214
+ * Clean old backups, keeping only the most recent N backups.
215
+ *
216
+ * @param keepCount - Number of recent backups to keep (default: 10)
217
+ * @returns Promise resolving to number of backups deleted
218
+ */
219
+ cleanOldBackups(keepCount?: number): Promise<number>;
220
+ /**
221
+ * Get the path to the backup directory.
222
+ */
223
+ getBackupDir(): string;
224
+ }
225
+ //# sourceMappingURL=IOManager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"IOManager.d.ts","sourceRoot":"","sources":["../../src/features/IOManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAIV,sBAAsB,EACtB,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EACb,aAAa,EACb,YAAY,EACZ,2BAA2B,EAC5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAkB5D;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,UAAU,GAAG,SAAS,CAAC;AAEhG;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAElE;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,8BAA8B;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,QAAQ,EAAE,cAAc,CAAC;IACzB,uCAAuC;IACvC,UAAU,EAAE,OAAO,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;CACd;AAMD;;;;;;;GAOG;AACH,qBAAa,SAAS;IAGR,OAAO,CAAC,OAAO;IAF3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEf,OAAO,EAAE,YAAY;IAUzC;;;;;;OAMG;IACH,WAAW,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM;IAqBxE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACG,0BAA0B,CAC9B,KAAK,EAAE,sBAAsB,EAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,YAAY,CAAC;IAuDxB;;;;;;;;;;;OAWG;YACW,YAAY;IA2C1B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,WAAW;IAoDnB,OAAO,CAAC,eAAe;IAuDvB,OAAO,CAAC,YAAY;IAqDpB,OAAO,CAAC,WAAW;IAiCnB,OAAO,CAAC,gBAAgB;IAwCxB,OAAO,CAAC,eAAe;IAyCvB;;;;;;;;;;;;OAYG;IACG,WAAW,CACf,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,aAAa,GAAE,aAAsB,EACrC,MAAM,GAAE,OAAe,EACvB,OAAO,CAAC,EAAE,2BAA2B,GACpC,OAAO,CAAC,YAAY,CAAC;IA6CxB,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,cAAc;IAuGtB,OAAO,CAAC,kBAAkB;YAgEZ,kBAAkB;IAuJhC;;OAEG;YACW,eAAe;IAQ7B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAW9B;;;;;;;;;;;;;;;;;;OAkBG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiF3E;;;;;;OAMG;IACG,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IA+D1C;;;;;;;;;;;;;;;;;;OAkBG;IACG,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAoCnE;;;;OAIG;IACG,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrD;;;;;OAKG;IACG,eAAe,CAAC,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB9D;;OAEG;IACH,YAAY,IAAI,MAAM;CAGvB"}