@danielsimonjr/memory-mcp 0.48.0 → 9.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +2000 -194
  3. package/dist/__tests__/file-path.test.js +7 -11
  4. package/dist/__tests__/knowledge-graph.test.js +3 -8
  5. package/dist/core/EntityManager.d.ts +266 -0
  6. package/dist/core/EntityManager.d.ts.map +1 -0
  7. package/dist/core/EntityManager.js +85 -133
  8. package/dist/core/GraphEventEmitter.d.ts +202 -0
  9. package/dist/core/GraphEventEmitter.d.ts.map +1 -0
  10. package/dist/core/GraphEventEmitter.js +346 -0
  11. package/dist/core/GraphStorage.d.ts +395 -0
  12. package/dist/core/GraphStorage.d.ts.map +1 -0
  13. package/dist/core/GraphStorage.js +643 -31
  14. package/dist/core/GraphTraversal.d.ts +141 -0
  15. package/dist/core/GraphTraversal.d.ts.map +1 -0
  16. package/dist/core/GraphTraversal.js +573 -0
  17. package/dist/core/HierarchyManager.d.ts +111 -0
  18. package/dist/core/HierarchyManager.d.ts.map +1 -0
  19. package/dist/{features → core}/HierarchyManager.js +14 -9
  20. package/dist/core/ManagerContext.d.ts +72 -0
  21. package/dist/core/ManagerContext.d.ts.map +1 -0
  22. package/dist/core/ManagerContext.js +118 -0
  23. package/dist/core/ObservationManager.d.ts +85 -0
  24. package/dist/core/ObservationManager.d.ts.map +1 -0
  25. package/dist/core/ObservationManager.js +51 -57
  26. package/dist/core/RelationManager.d.ts +131 -0
  27. package/dist/core/RelationManager.d.ts.map +1 -0
  28. package/dist/core/RelationManager.js +31 -7
  29. package/dist/core/SQLiteStorage.d.ts +354 -0
  30. package/dist/core/SQLiteStorage.d.ts.map +1 -0
  31. package/dist/core/SQLiteStorage.js +917 -0
  32. package/dist/core/StorageFactory.d.ts +45 -0
  33. package/dist/core/StorageFactory.d.ts.map +1 -0
  34. package/dist/core/StorageFactory.js +64 -0
  35. package/dist/core/TransactionManager.d.ts +464 -0
  36. package/dist/core/TransactionManager.d.ts.map +1 -0
  37. package/dist/core/TransactionManager.js +490 -13
  38. package/dist/core/index.d.ts +17 -0
  39. package/dist/core/index.d.ts.map +1 -0
  40. package/dist/core/index.js +12 -2
  41. package/dist/features/AnalyticsManager.d.ts +44 -0
  42. package/dist/features/AnalyticsManager.d.ts.map +1 -0
  43. package/dist/features/AnalyticsManager.js +3 -2
  44. package/dist/features/ArchiveManager.d.ts +133 -0
  45. package/dist/features/ArchiveManager.d.ts.map +1 -0
  46. package/dist/features/ArchiveManager.js +221 -14
  47. package/dist/features/CompressionManager.d.ts +117 -0
  48. package/dist/features/CompressionManager.d.ts.map +1 -0
  49. package/dist/features/CompressionManager.js +189 -20
  50. package/dist/features/IOManager.d.ts +225 -0
  51. package/dist/features/IOManager.d.ts.map +1 -0
  52. package/dist/features/IOManager.js +1041 -0
  53. package/dist/features/StreamingExporter.d.ts +123 -0
  54. package/dist/features/StreamingExporter.d.ts.map +1 -0
  55. package/dist/features/StreamingExporter.js +203 -0
  56. package/dist/features/TagManager.d.ts +147 -0
  57. package/dist/features/TagManager.d.ts.map +1 -0
  58. package/dist/features/index.d.ts +12 -0
  59. package/dist/features/index.d.ts.map +1 -0
  60. package/dist/features/index.js +5 -6
  61. package/dist/index.d.ts +9 -0
  62. package/dist/index.d.ts.map +1 -0
  63. package/dist/index.js +10 -10
  64. package/dist/memory.jsonl +1 -26
  65. package/dist/search/BasicSearch.d.ts +51 -0
  66. package/dist/search/BasicSearch.d.ts.map +1 -0
  67. package/dist/search/BasicSearch.js +9 -3
  68. package/dist/search/BooleanSearch.d.ts +98 -0
  69. package/dist/search/BooleanSearch.d.ts.map +1 -0
  70. package/dist/search/BooleanSearch.js +156 -9
  71. package/dist/search/EmbeddingService.d.ts +178 -0
  72. package/dist/search/EmbeddingService.d.ts.map +1 -0
  73. package/dist/search/EmbeddingService.js +358 -0
  74. package/dist/search/FuzzySearch.d.ts +118 -0
  75. package/dist/search/FuzzySearch.d.ts.map +1 -0
  76. package/dist/search/FuzzySearch.js +241 -25
  77. package/dist/search/QueryCostEstimator.d.ts +111 -0
  78. package/dist/search/QueryCostEstimator.d.ts.map +1 -0
  79. package/dist/search/QueryCostEstimator.js +355 -0
  80. package/dist/search/RankedSearch.d.ts +71 -0
  81. package/dist/search/RankedSearch.d.ts.map +1 -0
  82. package/dist/search/RankedSearch.js +54 -6
  83. package/dist/search/SavedSearchManager.d.ts +79 -0
  84. package/dist/search/SavedSearchManager.d.ts.map +1 -0
  85. package/dist/search/SearchFilterChain.d.ts +120 -0
  86. package/dist/search/SearchFilterChain.d.ts.map +1 -0
  87. package/dist/search/SearchFilterChain.js +2 -4
  88. package/dist/search/SearchManager.d.ts +326 -0
  89. package/dist/search/SearchManager.d.ts.map +1 -0
  90. package/dist/search/SearchManager.js +148 -0
  91. package/dist/search/SearchSuggestions.d.ts +27 -0
  92. package/dist/search/SearchSuggestions.d.ts.map +1 -0
  93. package/dist/search/SearchSuggestions.js +1 -1
  94. package/dist/search/SemanticSearch.d.ts +149 -0
  95. package/dist/search/SemanticSearch.d.ts.map +1 -0
  96. package/dist/search/SemanticSearch.js +323 -0
  97. package/dist/search/TFIDFEventSync.d.ts +85 -0
  98. package/dist/search/TFIDFEventSync.d.ts.map +1 -0
  99. package/dist/search/TFIDFEventSync.js +133 -0
  100. package/dist/search/TFIDFIndexManager.d.ts +151 -0
  101. package/dist/search/TFIDFIndexManager.d.ts.map +1 -0
  102. package/dist/search/TFIDFIndexManager.js +232 -17
  103. package/dist/search/VectorStore.d.ts +235 -0
  104. package/dist/search/VectorStore.d.ts.map +1 -0
  105. package/dist/search/VectorStore.js +311 -0
  106. package/dist/search/index.d.ts +21 -0
  107. package/dist/search/index.d.ts.map +1 -0
  108. package/dist/search/index.js +12 -0
  109. package/dist/server/MCPServer.d.ts +21 -0
  110. package/dist/server/MCPServer.d.ts.map +1 -0
  111. package/dist/server/MCPServer.js +4 -4
  112. package/dist/server/responseCompressor.d.ts +94 -0
  113. package/dist/server/responseCompressor.d.ts.map +1 -0
  114. package/dist/server/responseCompressor.js +127 -0
  115. package/dist/server/toolDefinitions.d.ts +27 -0
  116. package/dist/server/toolDefinitions.d.ts.map +1 -0
  117. package/dist/server/toolDefinitions.js +188 -17
  118. package/dist/server/toolHandlers.d.ts +41 -0
  119. package/dist/server/toolHandlers.d.ts.map +1 -0
  120. package/dist/server/toolHandlers.js +467 -75
  121. package/dist/types/index.d.ts +13 -0
  122. package/dist/types/index.d.ts.map +1 -0
  123. package/dist/types/index.js +1 -1
  124. package/dist/types/types.d.ts +1654 -0
  125. package/dist/types/types.d.ts.map +1 -0
  126. package/dist/types/types.js +9 -0
  127. package/dist/utils/compressedCache.d.ts +192 -0
  128. package/dist/utils/compressedCache.d.ts.map +1 -0
  129. package/dist/utils/compressedCache.js +309 -0
  130. package/dist/utils/compressionUtil.d.ts +214 -0
  131. package/dist/utils/compressionUtil.d.ts.map +1 -0
  132. package/dist/utils/compressionUtil.js +247 -0
  133. package/dist/utils/constants.d.ts +245 -0
  134. package/dist/utils/constants.d.ts.map +1 -0
  135. package/dist/utils/constants.js +124 -0
  136. package/dist/utils/entityUtils.d.ts +321 -0
  137. package/dist/utils/entityUtils.d.ts.map +1 -0
  138. package/dist/utils/entityUtils.js +434 -4
  139. package/dist/utils/errors.d.ts +95 -0
  140. package/dist/utils/errors.d.ts.map +1 -0
  141. package/dist/utils/errors.js +24 -0
  142. package/dist/utils/formatters.d.ts +145 -0
  143. package/dist/utils/formatters.d.ts.map +1 -0
  144. package/dist/utils/{paginationUtils.js → formatters.js} +54 -3
  145. package/dist/utils/index.d.ts +23 -0
  146. package/dist/utils/index.d.ts.map +1 -0
  147. package/dist/utils/index.js +69 -31
  148. package/dist/utils/indexes.d.ts +270 -0
  149. package/dist/utils/indexes.d.ts.map +1 -0
  150. package/dist/utils/indexes.js +526 -0
  151. package/dist/utils/logger.d.ts +24 -0
  152. package/dist/utils/logger.d.ts.map +1 -0
  153. package/dist/utils/operationUtils.d.ts +124 -0
  154. package/dist/utils/operationUtils.d.ts.map +1 -0
  155. package/dist/utils/operationUtils.js +175 -0
  156. package/dist/utils/parallelUtils.d.ts +72 -0
  157. package/dist/utils/parallelUtils.d.ts.map +1 -0
  158. package/dist/utils/parallelUtils.js +169 -0
  159. package/dist/utils/schemas.d.ts +374 -0
  160. package/dist/utils/schemas.d.ts.map +1 -0
  161. package/dist/utils/schemas.js +302 -2
  162. package/dist/utils/searchAlgorithms.d.ts +99 -0
  163. package/dist/utils/searchAlgorithms.d.ts.map +1 -0
  164. package/dist/utils/searchAlgorithms.js +167 -0
  165. package/dist/utils/searchCache.d.ts +108 -0
  166. package/dist/utils/searchCache.d.ts.map +1 -0
  167. package/dist/utils/taskScheduler.d.ts +290 -0
  168. package/dist/utils/taskScheduler.d.ts.map +1 -0
  169. package/dist/utils/taskScheduler.js +466 -0
  170. package/dist/workers/index.d.ts +12 -0
  171. package/dist/workers/index.d.ts.map +1 -0
  172. package/dist/workers/index.js +9 -0
  173. package/dist/workers/levenshteinWorker.d.ts +60 -0
  174. package/dist/workers/levenshteinWorker.d.ts.map +1 -0
  175. package/dist/workers/levenshteinWorker.js +98 -0
  176. package/package.json +17 -4
  177. package/dist/__tests__/edge-cases/edge-cases.test.js +0 -406
  178. package/dist/__tests__/integration/workflows.test.js +0 -449
  179. package/dist/__tests__/performance/benchmarks.test.js +0 -413
  180. package/dist/__tests__/unit/core/EntityManager.test.js +0 -334
  181. package/dist/__tests__/unit/core/GraphStorage.test.js +0 -205
  182. package/dist/__tests__/unit/core/RelationManager.test.js +0 -274
  183. package/dist/__tests__/unit/features/CompressionManager.test.js +0 -350
  184. package/dist/__tests__/unit/search/BasicSearch.test.js +0 -311
  185. package/dist/__tests__/unit/search/BooleanSearch.test.js +0 -432
  186. package/dist/__tests__/unit/search/FuzzySearch.test.js +0 -448
  187. package/dist/__tests__/unit/search/RankedSearch.test.js +0 -379
  188. package/dist/__tests__/unit/utils/levenshtein.test.js +0 -77
  189. package/dist/core/KnowledgeGraphManager.js +0 -423
  190. package/dist/features/BackupManager.js +0 -311
  191. package/dist/features/ExportManager.js +0 -305
  192. package/dist/features/ImportExportManager.js +0 -50
  193. package/dist/features/ImportManager.js +0 -328
  194. package/dist/memory-saved-searches.jsonl +0 -0
  195. package/dist/memory-tag-aliases.jsonl +0 -0
  196. package/dist/types/analytics.types.js +0 -6
  197. package/dist/types/entity.types.js +0 -7
  198. package/dist/types/import-export.types.js +0 -7
  199. package/dist/types/search.types.js +0 -7
  200. package/dist/types/tag.types.js +0 -6
  201. package/dist/utils/dateUtils.js +0 -89
  202. package/dist/utils/filterUtils.js +0 -155
  203. package/dist/utils/levenshtein.js +0 -62
  204. package/dist/utils/pathUtils.js +0 -115
  205. package/dist/utils/responseFormatter.js +0 -55
  206. package/dist/utils/tagUtils.js +0 -107
  207. package/dist/utils/tfidf.js +0 -90
  208. package/dist/utils/validationHelper.js +0 -99
  209. package/dist/utils/validationUtils.js +0 -109
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Archive Manager
3
+ *
4
+ * Handles archiving (removal) of entities based on criteria.
5
+ * Archives are stored as compressed files for space-efficient long-term storage.
6
+ * Extracted from EntityManager (Phase 4: Consolidate God Objects).
7
+ * Enhanced with brotli compression in Phase 3 Sprint 5.
8
+ *
9
+ * @module features/ArchiveManager
10
+ */
11
+ import type { LongRunningOperationOptions } from '../types/index.js';
12
+ import type { GraphStorage } from '../core/GraphStorage.js';
13
+ /**
14
+ * Criteria for archiving entities.
15
+ */
16
+ export interface ArchiveCriteria {
17
+ /** Entities older than this date (ISO 8601) */
18
+ olderThan?: string;
19
+ /** Entities with importance less than this value */
20
+ importanceLessThan?: number;
21
+ /** Entities with any of these tags */
22
+ tags?: string[];
23
+ }
24
+ /**
25
+ * Options for archive operations.
26
+ * Phase 9B: Extended with LongRunningOperationOptions.
27
+ */
28
+ export interface ArchiveOptions extends LongRunningOperationOptions {
29
+ /** Dry run mode - preview without making changes */
30
+ dryRun?: boolean;
31
+ /** Whether to save archived entities to a compressed file (default: true) */
32
+ saveToFile?: boolean;
33
+ }
34
+ /**
35
+ * Result of archive operation.
36
+ * Extends ArchiveResultExtended with compression statistics.
37
+ */
38
+ export interface ArchiveResult {
39
+ /** Number of entities archived */
40
+ archived: number;
41
+ /** Names of archived entities */
42
+ entityNames: string[];
43
+ /** Path to the archive file (if created) */
44
+ archivePath?: string;
45
+ /** Original size of archive data in bytes */
46
+ originalSize?: number;
47
+ /** Compressed size in bytes */
48
+ compressedSize?: number;
49
+ /** Compression ratio (compressedSize / originalSize). Lower is better. */
50
+ compressionRatio?: number;
51
+ }
52
+ /**
53
+ * Manages archive operations for the knowledge graph.
54
+ *
55
+ * Archives are stored as brotli-compressed files in the `.archives` directory.
56
+ * Maximum compression quality is used for optimal long-term storage.
57
+ */
58
+ export declare class ArchiveManager {
59
+ private storage;
60
+ private readonly archiveDir;
61
+ constructor(storage: GraphStorage);
62
+ /**
63
+ * Archive old or low-importance entities.
64
+ *
65
+ * Entities matching ANY of the criteria are archived:
66
+ * - lastModified older than olderThan date
67
+ * - importance less than importanceLessThan
68
+ * - has at least one tag from tags array
69
+ *
70
+ * By default, archived entities are saved to a compressed file before
71
+ * being removed from the active graph. Use `saveToFile: false` to
72
+ * skip creating the archive file.
73
+ *
74
+ * Phase 9B: Supports progress tracking and cancellation via options.
75
+ *
76
+ * @param criteria - Archiving criteria
77
+ * @param options - Archive options (dryRun, saveToFile, onProgress, signal)
78
+ * @returns Archive result with count, entity names, and compression stats
79
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
80
+ *
81
+ * @example
82
+ * ```typescript
83
+ * // Archive old entities with compression
84
+ * const result = await manager.archiveEntities({
85
+ * olderThan: '2023-01-01T00:00:00Z',
86
+ * importanceLessThan: 3
87
+ * });
88
+ * console.log(`Archived ${result.archived} entities`);
89
+ * console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
90
+ *
91
+ * // Preview without making changes
92
+ * const preview = await manager.archiveEntities(criteria, { dryRun: true });
93
+ *
94
+ * // With progress tracking and cancellation (Phase 9B)
95
+ * const controller = new AbortController();
96
+ * const result = await manager.archiveEntities(criteria, {
97
+ * signal: controller.signal,
98
+ * onProgress: (p) => console.log(`${p.percentage}% complete`),
99
+ * });
100
+ * ```
101
+ */
102
+ archiveEntities(criteria: ArchiveCriteria, options?: ArchiveOptions | boolean): Promise<ArchiveResult>;
103
+ /**
104
+ * Save entities to a compressed archive file.
105
+ *
106
+ * Creates a brotli-compressed file in the `.archives` directory
107
+ * with maximum compression quality for space efficiency.
108
+ *
109
+ * @param entities - Entities to archive
110
+ * @returns Archive file path and compression statistics
111
+ */
112
+ private saveToArchive;
113
+ /**
114
+ * List all available archives.
115
+ *
116
+ * @returns Array of archive information with compression details
117
+ */
118
+ listArchives(): Promise<Array<{
119
+ fileName: string;
120
+ filePath: string;
121
+ timestamp: string;
122
+ entityCount: number;
123
+ compressed: boolean;
124
+ originalSize?: number;
125
+ compressedSize?: number;
126
+ compressionRatio?: number;
127
+ }>>;
128
+ /**
129
+ * Get the path to the archives directory.
130
+ */
131
+ getArchiveDir(): string;
132
+ }
133
+ //# sourceMappingURL=ArchiveManager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ArchiveManager.d.ts","sourceRoot":"","sources":["../../src/features/ArchiveManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAU,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAS5D;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,oDAAoD;IACpD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+BAA+B;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,qBAAa,cAAc;IAGb,OAAO,CAAC,OAAO;IAF3B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAEhB,OAAO,EAAE,YAAY;IAMzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACG,eAAe,CACnB,QAAQ,EAAE,eAAe,EACzB,OAAO,GAAE,cAAc,GAAG,OAAY,GACrC,OAAO,CAAC,aAAa,CAAC;IAuIzB;;;;;;;;OAQG;YACW,aAAa;IAmD3B;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC;QAClC,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,OAAO,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IA6DH;;OAEG;IACH,aAAa,IAAI,MAAM;CAGxB"}
@@ -1,17 +1,30 @@
1
1
  /**
2
2
  * Archive Manager
3
3
  *
4
- * Archives old or low-importance entities to reduce active graph size.
4
+ * Handles archiving (removal) of entities based on criteria.
5
+ * Archives are stored as compressed files for space-efficient long-term storage.
6
+ * Extracted from EntityManager (Phase 4: Consolidate God Objects).
7
+ * Enhanced with brotli compression in Phase 3 Sprint 5.
5
8
  *
6
9
  * @module features/ArchiveManager
7
10
  */
11
+ import { promises as fs } from 'fs';
12
+ import { dirname, join } from 'path';
13
+ import { compress, COMPRESSION_CONFIG, checkCancellation, createProgressReporter, createProgress, } from '../utils/index.js';
8
14
  /**
9
- * Manages entity archival based on age, importance, and tags.
15
+ * Manages archive operations for the knowledge graph.
16
+ *
17
+ * Archives are stored as brotli-compressed files in the `.archives` directory.
18
+ * Maximum compression quality is used for optimal long-term storage.
10
19
  */
11
20
  export class ArchiveManager {
12
21
  storage;
22
+ archiveDir;
13
23
  constructor(storage) {
14
24
  this.storage = storage;
25
+ const filePath = this.storage.getFilePath();
26
+ const dir = dirname(filePath);
27
+ this.archiveDir = join(dir, '.archives');
15
28
  }
16
29
  /**
17
30
  * Archive old or low-importance entities.
@@ -21,16 +34,58 @@ export class ArchiveManager {
21
34
  * - importance less than importanceLessThan
22
35
  * - has at least one tag from tags array
23
36
  *
24
- * Archived entities and their relations are removed from the graph.
37
+ * By default, archived entities are saved to a compressed file before
38
+ * being removed from the active graph. Use `saveToFile: false` to
39
+ * skip creating the archive file.
40
+ *
41
+ * Phase 9B: Supports progress tracking and cancellation via options.
25
42
  *
26
43
  * @param criteria - Archiving criteria
27
- * @param dryRun - If true, preview what would be archived without making changes
28
- * @returns Archive result with count and entity names
44
+ * @param options - Archive options (dryRun, saveToFile, onProgress, signal)
45
+ * @returns Archive result with count, entity names, and compression stats
46
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
47
+ *
48
+ * @example
49
+ * ```typescript
50
+ * // Archive old entities with compression
51
+ * const result = await manager.archiveEntities({
52
+ * olderThan: '2023-01-01T00:00:00Z',
53
+ * importanceLessThan: 3
54
+ * });
55
+ * console.log(`Archived ${result.archived} entities`);
56
+ * console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
57
+ *
58
+ * // Preview without making changes
59
+ * const preview = await manager.archiveEntities(criteria, { dryRun: true });
60
+ *
61
+ * // With progress tracking and cancellation (Phase 9B)
62
+ * const controller = new AbortController();
63
+ * const result = await manager.archiveEntities(criteria, {
64
+ * signal: controller.signal,
65
+ * onProgress: (p) => console.log(`${p.percentage}% complete`),
66
+ * });
67
+ * ```
29
68
  */
30
- async archiveEntities(criteria, dryRun = false) {
31
- const graph = await this.storage.loadGraph();
69
+ async archiveEntities(criteria, options = {}) {
70
+ // Handle legacy boolean argument (backward compatibility)
71
+ const opts = typeof options === 'boolean'
72
+ ? { dryRun: options, saveToFile: true }
73
+ : { saveToFile: true, ...options };
74
+ // Check for early cancellation
75
+ checkCancellation(opts.signal, 'archiveEntities');
76
+ // Setup progress reporter
77
+ const reportProgress = createProgressReporter(opts.onProgress);
78
+ reportProgress?.(createProgress(0, 100, 'archiveEntities'));
79
+ // Use read-only graph for analysis
80
+ const readGraph = await this.storage.loadGraph();
32
81
  const toArchive = [];
33
- for (const entity of graph.entities) {
82
+ const totalEntities = readGraph.entities.length;
83
+ let processedEntities = 0;
84
+ // Phase 1: Identify entities to archive (0-40% progress)
85
+ reportProgress?.(createProgress(5, 100, 'analyzing entities'));
86
+ for (const entity of readGraph.entities) {
87
+ // Check for cancellation periodically
88
+ checkCancellation(opts.signal, 'archiveEntities');
34
89
  let shouldArchive = false;
35
90
  // Check age criteria
36
91
  if (criteria.olderThan && entity.lastModified) {
@@ -58,17 +113,169 @@ export class ArchiveManager {
58
113
  if (shouldArchive) {
59
114
  toArchive.push(entity);
60
115
  }
116
+ processedEntities++;
117
+ // Map analysis progress (0-100%) to overall progress (0-40%)
118
+ const analysisProgress = totalEntities > 0 ? Math.round((processedEntities / totalEntities) * 40) : 40;
119
+ reportProgress?.(createProgress(analysisProgress, 100, 'analyzing entities'));
120
+ }
121
+ reportProgress?.(createProgress(40, 100, 'analysis complete'));
122
+ // Dry run - return preview without changes
123
+ if (opts.dryRun) {
124
+ reportProgress?.(createProgress(100, 100, 'archiveEntities'));
125
+ return {
126
+ archived: toArchive.length,
127
+ entityNames: toArchive.map(e => e.name),
128
+ };
129
+ }
130
+ // No entities to archive
131
+ if (toArchive.length === 0) {
132
+ reportProgress?.(createProgress(100, 100, 'archiveEntities'));
133
+ return {
134
+ archived: 0,
135
+ entityNames: [],
136
+ };
137
+ }
138
+ // Check for cancellation before archiving
139
+ checkCancellation(opts.signal, 'archiveEntities');
140
+ // Phase 2: Save to compressed archive file (40-80% progress)
141
+ let archivePath;
142
+ let originalSize;
143
+ let compressedSize;
144
+ let compressionRatio;
145
+ if (opts.saveToFile) {
146
+ reportProgress?.(createProgress(50, 100, 'compressing archive'));
147
+ const archiveResult = await this.saveToArchive(toArchive);
148
+ archivePath = archiveResult.archivePath;
149
+ originalSize = archiveResult.originalSize;
150
+ compressedSize = archiveResult.compressedSize;
151
+ compressionRatio = archiveResult.compressionRatio;
152
+ reportProgress?.(createProgress(80, 100, 'archive saved'));
61
153
  }
62
- if (!dryRun && toArchive.length > 0) {
63
- // Remove archived entities from main graph
64
- const archiveNames = new Set(toArchive.map(e => e.name));
65
- graph.entities = graph.entities.filter(e => !archiveNames.has(e.name));
66
- graph.relations = graph.relations.filter(r => !archiveNames.has(r.from) && !archiveNames.has(r.to));
67
- await this.storage.saveGraph(graph);
154
+ else {
155
+ reportProgress?.(createProgress(80, 100, 'skipped archive file'));
68
156
  }
157
+ // Check for cancellation before graph modification
158
+ checkCancellation(opts.signal, 'archiveEntities');
159
+ // Phase 3: Remove from main graph (80-100% progress)
160
+ reportProgress?.(createProgress(85, 100, 'updating graph'));
161
+ // Get mutable copy for write operation
162
+ const graph = await this.storage.getGraphForMutation();
163
+ // Remove archived entities from main graph
164
+ const archiveNames = new Set(toArchive.map(e => e.name));
165
+ graph.entities = graph.entities.filter(e => !archiveNames.has(e.name));
166
+ graph.relations = graph.relations.filter(r => !archiveNames.has(r.from) && !archiveNames.has(r.to));
167
+ await this.storage.saveGraph(graph);
168
+ // Report completion
169
+ reportProgress?.(createProgress(100, 100, 'archiveEntities'));
69
170
  return {
70
171
  archived: toArchive.length,
71
172
  entityNames: toArchive.map(e => e.name),
173
+ archivePath,
174
+ originalSize,
175
+ compressedSize,
176
+ compressionRatio,
177
+ };
178
+ }
179
+ /**
180
+ * Save entities to a compressed archive file.
181
+ *
182
+ * Creates a brotli-compressed file in the `.archives` directory
183
+ * with maximum compression quality for space efficiency.
184
+ *
185
+ * @param entities - Entities to archive
186
+ * @returns Archive file path and compression statistics
187
+ */
188
+ async saveToArchive(entities) {
189
+ // Ensure archive directory exists
190
+ await fs.mkdir(this.archiveDir, { recursive: true });
191
+ // Generate timestamp-based filename
192
+ const timestamp = new Date().toISOString()
193
+ .replace(/:/g, '-')
194
+ .replace(/\./g, '-')
195
+ .replace('T', '_')
196
+ .replace('Z', '');
197
+ const archivePath = join(this.archiveDir, `archive_${timestamp}.jsonl.br`);
198
+ // Serialize entities to JSONL format
199
+ const content = entities.map(e => JSON.stringify(e)).join('\n');
200
+ // Compress with maximum quality for archives
201
+ const compressionResult = await compress(content, {
202
+ quality: COMPRESSION_CONFIG.BROTLI_QUALITY_ARCHIVE,
203
+ mode: 'text',
204
+ });
205
+ // Write compressed archive
206
+ await fs.writeFile(archivePath, compressionResult.compressed);
207
+ // Write metadata file
208
+ const metadataPath = `${archivePath}.meta.json`;
209
+ const metadata = {
210
+ timestamp: new Date().toISOString(),
211
+ entityCount: entities.length,
212
+ entityNames: entities.map(e => e.name),
213
+ compressed: true,
214
+ compressionFormat: 'brotli',
215
+ originalSize: compressionResult.originalSize,
216
+ compressedSize: compressionResult.compressedSize,
217
+ compressionRatio: compressionResult.ratio,
218
+ };
219
+ await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
220
+ return {
221
+ archivePath,
222
+ originalSize: compressionResult.originalSize,
223
+ compressedSize: compressionResult.compressedSize,
224
+ compressionRatio: compressionResult.ratio,
72
225
  };
73
226
  }
227
+ /**
228
+ * List all available archives.
229
+ *
230
+ * @returns Array of archive information with compression details
231
+ */
232
+ async listArchives() {
233
+ try {
234
+ try {
235
+ await fs.access(this.archiveDir);
236
+ }
237
+ catch {
238
+ return [];
239
+ }
240
+ const files = await fs.readdir(this.archiveDir);
241
+ const archiveFiles = files.filter(f => f.startsWith('archive_') &&
242
+ (f.endsWith('.jsonl') || f.endsWith('.jsonl.br')) &&
243
+ !f.endsWith('.meta.json'));
244
+ const archives = [];
245
+ for (const fileName of archiveFiles) {
246
+ const filePath = join(this.archiveDir, fileName);
247
+ const metadataPath = `${filePath}.meta.json`;
248
+ try {
249
+ const metadataContent = await fs.readFile(metadataPath, 'utf-8');
250
+ const metadata = JSON.parse(metadataContent);
251
+ archives.push({
252
+ fileName,
253
+ filePath,
254
+ timestamp: metadata.timestamp,
255
+ entityCount: metadata.entityCount,
256
+ compressed: metadata.compressed ?? fileName.endsWith('.br'),
257
+ originalSize: metadata.originalSize,
258
+ compressedSize: metadata.compressedSize,
259
+ compressionRatio: metadata.compressionRatio,
260
+ });
261
+ }
262
+ catch {
263
+ // Skip archives without valid metadata
264
+ continue;
265
+ }
266
+ }
267
+ // Sort by timestamp (newest first)
268
+ archives.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
269
+ return archives;
270
+ }
271
+ catch {
272
+ return [];
273
+ }
274
+ }
275
+ /**
276
+ * Get the path to the archives directory.
277
+ */
278
+ getArchiveDir() {
279
+ return this.archiveDir;
280
+ }
74
281
  }
@@ -0,0 +1,117 @@
1
+ /**
2
+ * Compression Manager
3
+ *
4
+ * Handles duplicate detection, entity merging, and graph compression.
5
+ * Extracted from SearchManager (Phase 4: Consolidate God Objects).
6
+ *
7
+ * @module features/CompressionManager
8
+ */
9
+ import type { Entity, CompressionResult, KnowledgeGraph, LongRunningOperationOptions } from '../types/index.js';
10
+ import type { GraphStorage } from '../core/GraphStorage.js';
11
+ /**
12
+ * Manages compression operations for the knowledge graph.
13
+ */
14
+ export declare class CompressionManager {
15
+ private storage;
16
+ constructor(storage: GraphStorage);
17
+ /**
18
+ * Prepare an entity for efficient similarity comparisons.
19
+ * Pre-computes all normalized data to avoid repeated computation.
20
+ *
21
+ * @param entity - The entity to prepare
22
+ * @returns PreparedEntity with pre-computed data
23
+ */
24
+ private prepareEntity;
25
+ /**
26
+ * Prepare multiple entities for efficient similarity comparisons.
27
+ * Use this before batch comparison operations.
28
+ *
29
+ * @param entities - Entities to prepare
30
+ * @returns Map of entity name to PreparedEntity
31
+ */
32
+ private prepareEntities;
33
+ /**
34
+ * Calculate similarity between two entities using multiple heuristics.
35
+ *
36
+ * Uses configurable weights defined in SIMILARITY_WEIGHTS constant.
37
+ * See SIMILARITY_WEIGHTS for the breakdown of scoring factors.
38
+ *
39
+ * NOTE: For batch comparisons, use prepareEntities() + calculatePreparedSimilarity() for better performance.
40
+ *
41
+ * @param e1 - First entity
42
+ * @param e2 - Second entity
43
+ * @returns Similarity score from 0 (completely different) to 1 (identical)
44
+ */
45
+ calculateEntitySimilarity(e1: Entity, e2: Entity): number;
46
+ /**
47
+ * Efficiently calculate intersection size of two Sets without creating a new Set.
48
+ * Iterates over the smaller set for O(min(m,n)) complexity.
49
+ */
50
+ private setIntersectionSize;
51
+ /**
52
+ * Calculate similarity between two prepared entities.
53
+ * OPTIMIZED: Uses pre-computed Sets to avoid O(n) set creation per comparison.
54
+ *
55
+ * @param p1 - First prepared entity
56
+ * @param p2 - Second prepared entity
57
+ * @returns Similarity score from 0 (completely different) to 1 (identical)
58
+ */
59
+ private calculatePreparedSimilarity;
60
+ /**
61
+ * Find duplicate entities in the graph based on similarity threshold.
62
+ *
63
+ * OPTIMIZED: Uses bucketing strategies to reduce O(n²) comparisons:
64
+ * 1. Buckets entities by entityType (only compare same types)
65
+ * 2. Within each type, buckets by name prefix (first 2 chars normalized)
66
+ * 3. Only compares entities within same or adjacent buckets
67
+ *
68
+ * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
69
+ *
70
+ * Complexity: O(n·k) where k is average bucket size (typically << n)
71
+ *
72
+ * @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
73
+ * @param options - Optional progress/cancellation options (Phase 9B)
74
+ * @returns Array of duplicate groups (each group has similar entities)
75
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
76
+ */
77
+ findDuplicates(threshold?: number, options?: LongRunningOperationOptions): Promise<string[][]>;
78
+ /**
79
+ * Merge a group of entities into a single entity.
80
+ *
81
+ * Merging strategy:
82
+ * - First entity is kept (or renamed to targetName)
83
+ * - Observations: Union of all observations
84
+ * - Tags: Union of all tags
85
+ * - Importance: Maximum importance value
86
+ * - createdAt: Earliest date
87
+ * - lastModified: Current timestamp
88
+ * - Relations: Redirected to kept entity, duplicates removed
89
+ *
90
+ * @param entityNames - Names of entities to merge (first one is kept)
91
+ * @param targetName - Optional new name for merged entity (default: first entity name)
92
+ * @param options - Optional configuration
93
+ * @param options.graph - Pre-loaded graph to use (avoids reload)
94
+ * @param options.skipSave - If true, don't save (caller will save)
95
+ * @returns The merged entity
96
+ * @throws {InsufficientEntitiesError} If less than 2 entities provided
97
+ * @throws {EntityNotFoundError} If any entity not found
98
+ */
99
+ mergeEntities(entityNames: string[], targetName?: string, options?: {
100
+ graph?: KnowledgeGraph;
101
+ skipSave?: boolean;
102
+ }): Promise<Entity>;
103
+ /**
104
+ * Compress the knowledge graph by finding and merging duplicates.
105
+ * OPTIMIZED: Loads graph once, performs all merges, saves once.
106
+ *
107
+ * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
108
+ *
109
+ * @param threshold - Similarity threshold for duplicate detection (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
110
+ * @param dryRun - If true, only report what would be compressed without applying changes
111
+ * @param options - Optional progress/cancellation options (Phase 9B)
112
+ * @returns Compression result with statistics
113
+ * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
114
+ */
115
+ compressGraph(threshold?: number, dryRun?: boolean, options?: LongRunningOperationOptions): Promise<CompressionResult>;
116
+ }
117
+ //# sourceMappingURL=CompressionManager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"CompressionManager.d.ts","sourceRoot":"","sources":["../../src/features/CompressionManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAY,iBAAiB,EAAE,cAAc,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAC1H,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AA2B5D;;GAEG;AACH,qBAAa,kBAAkB;IACjB,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,YAAY;IAEzC;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAQvB;;;;;;;;;;;OAWG;IACH,yBAAyB,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM;IAwCzD;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IAoCnC;;;;;;;;;;;;;;;;OAgBG;IACG,cAAc,CAClB,SAAS,GAAE,MAAoC,EAC/C,OAAO,CAAC,EAAE,2BAA2B,GACpC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IA0GtB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EAAE,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE;QACP,KAAK,CAAC,EAAE,cAAc,CAAC;QACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACf,GACL,OAAO,CAAC,MAAM,CAAC;IA8FlB;;;;;;;;;;;OAWG;IACG,aAAa,CACjB,SAAS,GAAE,MAAoC,EAC/C,MAAM,GAAE,OAAe,EACvB,OAAO,CAAC,EAAE,2BAA2B,GACpC,OAAO,CAAC,iBAAiB,CAAC;CA0G9B"}