@danielsimonjr/memory-mcp 11.0.1 → 11.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/dist/core/EntityManager.d.ts +10 -15
- package/dist/core/EntityManager.d.ts.map +1 -1
- package/dist/core/EntityManager.js +21 -54
- package/dist/core/GraphStorage.d.ts +0 -51
- package/dist/core/GraphStorage.d.ts.map +1 -1
- package/dist/core/GraphStorage.js +2 -79
- package/dist/core/GraphTraversal.d.ts +2 -7
- package/dist/core/GraphTraversal.d.ts.map +1 -1
- package/dist/core/GraphTraversal.js +2 -19
- package/dist/core/ManagerContext.d.ts +0 -4
- package/dist/core/ManagerContext.d.ts.map +1 -1
- package/dist/core/ManagerContext.js +2 -12
- package/dist/core/RelationManager.d.ts.map +1 -1
- package/dist/core/RelationManager.js +4 -5
- package/dist/core/SQLiteStorage.d.ts.map +1 -1
- package/dist/core/SQLiteStorage.js +2 -3
- package/dist/core/TransactionManager.d.ts +2 -207
- package/dist/core/TransactionManager.d.ts.map +1 -1
- package/dist/core/TransactionManager.js +6 -482
- package/dist/core/index.d.ts +1 -2
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -3
- package/dist/features/ArchiveManager.d.ts +2 -14
- package/dist/features/ArchiveManager.d.ts.map +1 -1
- package/dist/features/ArchiveManager.js +3 -44
- package/dist/features/CompressionManager.d.ts +4 -14
- package/dist/features/CompressionManager.d.ts.map +1 -1
- package/dist/features/CompressionManager.js +9 -74
- package/dist/features/IOManager.d.ts +2 -6
- package/dist/features/IOManager.d.ts.map +1 -1
- package/dist/features/IOManager.js +10 -105
- package/dist/features/StreamingExporter.d.ts +4 -27
- package/dist/features/StreamingExporter.d.ts.map +1 -1
- package/dist/features/StreamingExporter.js +4 -65
- package/dist/features/index.d.ts +0 -2
- package/dist/features/index.d.ts.map +1 -1
- package/dist/features/index.js +0 -3
- package/dist/search/EmbeddingService.d.ts +9 -108
- package/dist/search/EmbeddingService.d.ts.map +1 -1
- package/dist/search/EmbeddingService.js +15 -187
- package/dist/search/FuzzySearch.js +1 -1
- package/dist/search/SavedSearchManager.d.ts.map +1 -1
- package/dist/search/SavedSearchManager.js +2 -3
- package/dist/search/SearchManager.d.ts +1 -42
- package/dist/search/SearchManager.d.ts.map +1 -1
- package/dist/search/SearchManager.js +0 -115
- package/dist/search/SemanticSearch.d.ts +1 -4
- package/dist/search/SemanticSearch.d.ts.map +1 -1
- package/dist/search/SemanticSearch.js +2 -12
- package/dist/search/TFIDFIndexManager.d.ts +0 -88
- package/dist/search/TFIDFIndexManager.d.ts.map +1 -1
- package/dist/search/TFIDFIndexManager.js +0 -217
- package/dist/search/index.d.ts +1 -18
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +1 -32
- package/dist/server/MCPServer.d.ts.map +1 -1
- package/dist/server/MCPServer.js +4 -1
- package/dist/server/responseCompressor.js +5 -5
- package/dist/server/toolDefinitions.d.ts.map +1 -1
- package/dist/server/toolDefinitions.js +5 -1
- package/dist/server/toolHandlers.d.ts +9 -5
- package/dist/server/toolHandlers.d.ts.map +1 -1
- package/dist/server/toolHandlers.js +23 -8
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/types.d.ts +2 -579
- package/dist/types/types.d.ts.map +1 -1
- package/dist/utils/compressedCache.d.ts +0 -29
- package/dist/utils/compressedCache.d.ts.map +1 -1
- package/dist/utils/compressedCache.js +0 -39
- package/dist/utils/entityUtils.d.ts +1 -59
- package/dist/utils/entityUtils.d.ts.map +1 -1
- package/dist/utils/entityUtils.js +3 -113
- package/dist/utils/errors.d.ts +0 -18
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +0 -24
- package/dist/utils/index.d.ts +2 -6
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -14
- package/dist/utils/logger.d.ts +0 -7
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +2 -9
- package/dist/utils/parallelUtils.d.ts +1 -5
- package/dist/utils/parallelUtils.d.ts.map +1 -1
- package/dist/utils/parallelUtils.js +1 -23
- package/dist/utils/schemas.d.ts +16 -16
- package/dist/utils/schemas.d.ts.map +1 -1
- package/dist/utils/schemas.js +12 -12
- package/dist/utils/taskScheduler.d.ts +0 -4
- package/dist/utils/taskScheduler.d.ts.map +1 -1
- package/dist/utils/taskScheduler.js +1 -21
- package/dist/workers/WorkerPool.d.ts +81 -0
- package/dist/workers/WorkerPool.d.ts.map +1 -0
- package/dist/workers/WorkerPool.js +121 -0
- package/dist/workers/index.d.ts +1 -1
- package/dist/workers/index.d.ts.map +1 -1
- package/dist/workers/levenshteinWorker.js +1 -1
- package/package.json +1 -4
- package/dist/__tests__/file-path.test.js +0 -119
- package/dist/__tests__/knowledge-graph.test.js +0 -318
- package/dist/core/GraphEventEmitter.d.ts +0 -202
- package/dist/core/GraphEventEmitter.d.ts.map +0 -1
- package/dist/core/GraphEventEmitter.js +0 -346
- package/dist/features/KeywordExtractor.d.ts +0 -61
- package/dist/features/KeywordExtractor.d.ts.map +0 -1
- package/dist/features/KeywordExtractor.js +0 -126
- package/dist/features/ObservationNormalizer.d.ts +0 -90
- package/dist/features/ObservationNormalizer.d.ts.map +0 -1
- package/dist/features/ObservationNormalizer.js +0 -193
- package/dist/memory.jsonl +0 -1
- package/dist/search/BM25Search.d.ts +0 -148
- package/dist/search/BM25Search.d.ts.map +0 -1
- package/dist/search/BM25Search.js +0 -339
- package/dist/search/EarlyTerminationManager.d.ts +0 -140
- package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
- package/dist/search/EarlyTerminationManager.js +0 -279
- package/dist/search/EmbeddingCache.d.ts +0 -175
- package/dist/search/EmbeddingCache.d.ts.map +0 -1
- package/dist/search/EmbeddingCache.js +0 -246
- package/dist/search/HybridScorer.d.ts +0 -181
- package/dist/search/HybridScorer.d.ts.map +0 -1
- package/dist/search/HybridScorer.js +0 -257
- package/dist/search/HybridSearchManager.d.ts +0 -80
- package/dist/search/HybridSearchManager.d.ts.map +0 -1
- package/dist/search/HybridSearchManager.js +0 -187
- package/dist/search/IncrementalIndexer.d.ts +0 -201
- package/dist/search/IncrementalIndexer.d.ts.map +0 -1
- package/dist/search/IncrementalIndexer.js +0 -342
- package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
- package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
- package/dist/search/OptimizedInvertedIndex.js +0 -358
- package/dist/search/ParallelSearchExecutor.d.ts +0 -172
- package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
- package/dist/search/ParallelSearchExecutor.js +0 -309
- package/dist/search/QuantizedVectorStore.d.ts +0 -171
- package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
- package/dist/search/QuantizedVectorStore.js +0 -307
- package/dist/search/QueryAnalyzer.d.ts +0 -76
- package/dist/search/QueryAnalyzer.d.ts.map +0 -1
- package/dist/search/QueryAnalyzer.js +0 -227
- package/dist/search/QueryCostEstimator.d.ts +0 -244
- package/dist/search/QueryCostEstimator.d.ts.map +0 -1
- package/dist/search/QueryCostEstimator.js +0 -652
- package/dist/search/QueryPlanCache.d.ts +0 -220
- package/dist/search/QueryPlanCache.d.ts.map +0 -1
- package/dist/search/QueryPlanCache.js +0 -379
- package/dist/search/QueryPlanner.d.ts +0 -58
- package/dist/search/QueryPlanner.d.ts.map +0 -1
- package/dist/search/QueryPlanner.js +0 -137
- package/dist/search/ReflectionManager.d.ts +0 -120
- package/dist/search/ReflectionManager.d.ts.map +0 -1
- package/dist/search/ReflectionManager.js +0 -231
- package/dist/search/SymbolicSearch.d.ts +0 -61
- package/dist/search/SymbolicSearch.d.ts.map +0 -1
- package/dist/search/SymbolicSearch.js +0 -163
- package/dist/search/TFIDFEventSync.d.ts +0 -85
- package/dist/search/TFIDFEventSync.d.ts.map +0 -1
- package/dist/search/TFIDFEventSync.js +0 -133
- package/dist/utils/BatchProcessor.d.ts +0 -271
- package/dist/utils/BatchProcessor.d.ts.map +0 -1
- package/dist/utils/BatchProcessor.js +0 -376
- package/dist/utils/MemoryMonitor.d.ts +0 -176
- package/dist/utils/MemoryMonitor.d.ts.map +0 -1
- package/dist/utils/MemoryMonitor.js +0 -305
- package/dist/utils/WorkerPoolManager.d.ts +0 -233
- package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
- package/dist/utils/WorkerPoolManager.js +0 -420
- package/dist/utils/operationUtils.d.ts +0 -124
- package/dist/utils/operationUtils.d.ts.map +0 -1
- package/dist/utils/operationUtils.js +0 -175
- package/dist/vitest.config.js +0 -13
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { promises as fs } from 'fs';
|
|
12
12
|
import { dirname, join } from 'path';
|
|
13
|
-
import { compress, COMPRESSION_CONFIG
|
|
13
|
+
import { compress, COMPRESSION_CONFIG } from '../utils/index.js';
|
|
14
14
|
/**
|
|
15
15
|
* Manages archive operations for the knowledge graph.
|
|
16
16
|
*
|
|
@@ -38,12 +38,9 @@ export class ArchiveManager {
|
|
|
38
38
|
* being removed from the active graph. Use `saveToFile: false` to
|
|
39
39
|
* skip creating the archive file.
|
|
40
40
|
*
|
|
41
|
-
* Phase 9B: Supports progress tracking and cancellation via options.
|
|
42
|
-
*
|
|
43
41
|
* @param criteria - Archiving criteria
|
|
44
|
-
* @param options - Archive options (dryRun, saveToFile
|
|
42
|
+
* @param options - Archive options (dryRun, saveToFile)
|
|
45
43
|
* @returns Archive result with count, entity names, and compression stats
|
|
46
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
47
44
|
*
|
|
48
45
|
* @example
|
|
49
46
|
* ```typescript
|
|
@@ -57,13 +54,6 @@ export class ArchiveManager {
|
|
|
57
54
|
*
|
|
58
55
|
* // Preview without making changes
|
|
59
56
|
* const preview = await manager.archiveEntities(criteria, { dryRun: true });
|
|
60
|
-
*
|
|
61
|
-
* // With progress tracking and cancellation (Phase 9B)
|
|
62
|
-
* const controller = new AbortController();
|
|
63
|
-
* const result = await manager.archiveEntities(criteria, {
|
|
64
|
-
* signal: controller.signal,
|
|
65
|
-
* onProgress: (p) => console.log(`${p.percentage}% complete`),
|
|
66
|
-
* });
|
|
67
57
|
* ```
|
|
68
58
|
*/
|
|
69
59
|
async archiveEntities(criteria, options = {}) {
|
|
@@ -71,21 +61,10 @@ export class ArchiveManager {
|
|
|
71
61
|
const opts = typeof options === 'boolean'
|
|
72
62
|
? { dryRun: options, saveToFile: true }
|
|
73
63
|
: { saveToFile: true, ...options };
|
|
74
|
-
// Check for early cancellation
|
|
75
|
-
checkCancellation(opts.signal, 'archiveEntities');
|
|
76
|
-
// Setup progress reporter
|
|
77
|
-
const reportProgress = createProgressReporter(opts.onProgress);
|
|
78
|
-
reportProgress?.(createProgress(0, 100, 'archiveEntities'));
|
|
79
64
|
// Use read-only graph for analysis
|
|
80
65
|
const readGraph = await this.storage.loadGraph();
|
|
81
66
|
const toArchive = [];
|
|
82
|
-
const totalEntities = readGraph.entities.length;
|
|
83
|
-
let processedEntities = 0;
|
|
84
|
-
// Phase 1: Identify entities to archive (0-40% progress)
|
|
85
|
-
reportProgress?.(createProgress(5, 100, 'analyzing entities'));
|
|
86
67
|
for (const entity of readGraph.entities) {
|
|
87
|
-
// Check for cancellation periodically
|
|
88
|
-
checkCancellation(opts.signal, 'archiveEntities');
|
|
89
68
|
let shouldArchive = false;
|
|
90
69
|
// Check age criteria
|
|
91
70
|
if (criteria.olderThan && entity.lastModified) {
|
|
@@ -113,15 +92,9 @@ export class ArchiveManager {
|
|
|
113
92
|
if (shouldArchive) {
|
|
114
93
|
toArchive.push(entity);
|
|
115
94
|
}
|
|
116
|
-
processedEntities++;
|
|
117
|
-
// Map analysis progress (0-100%) to overall progress (0-40%)
|
|
118
|
-
const analysisProgress = totalEntities > 0 ? Math.round((processedEntities / totalEntities) * 40) : 40;
|
|
119
|
-
reportProgress?.(createProgress(analysisProgress, 100, 'analyzing entities'));
|
|
120
95
|
}
|
|
121
|
-
reportProgress?.(createProgress(40, 100, 'analysis complete'));
|
|
122
96
|
// Dry run - return preview without changes
|
|
123
97
|
if (opts.dryRun) {
|
|
124
|
-
reportProgress?.(createProgress(100, 100, 'archiveEntities'));
|
|
125
98
|
return {
|
|
126
99
|
archived: toArchive.length,
|
|
127
100
|
entityNames: toArchive.map(e => e.name),
|
|
@@ -129,35 +102,23 @@ export class ArchiveManager {
|
|
|
129
102
|
}
|
|
130
103
|
// No entities to archive
|
|
131
104
|
if (toArchive.length === 0) {
|
|
132
|
-
reportProgress?.(createProgress(100, 100, 'archiveEntities'));
|
|
133
105
|
return {
|
|
134
106
|
archived: 0,
|
|
135
107
|
entityNames: [],
|
|
136
108
|
};
|
|
137
109
|
}
|
|
138
|
-
//
|
|
139
|
-
checkCancellation(opts.signal, 'archiveEntities');
|
|
140
|
-
// Phase 2: Save to compressed archive file (40-80% progress)
|
|
110
|
+
// Save to compressed archive file
|
|
141
111
|
let archivePath;
|
|
142
112
|
let originalSize;
|
|
143
113
|
let compressedSize;
|
|
144
114
|
let compressionRatio;
|
|
145
115
|
if (opts.saveToFile) {
|
|
146
|
-
reportProgress?.(createProgress(50, 100, 'compressing archive'));
|
|
147
116
|
const archiveResult = await this.saveToArchive(toArchive);
|
|
148
117
|
archivePath = archiveResult.archivePath;
|
|
149
118
|
originalSize = archiveResult.originalSize;
|
|
150
119
|
compressedSize = archiveResult.compressedSize;
|
|
151
120
|
compressionRatio = archiveResult.compressionRatio;
|
|
152
|
-
reportProgress?.(createProgress(80, 100, 'archive saved'));
|
|
153
|
-
}
|
|
154
|
-
else {
|
|
155
|
-
reportProgress?.(createProgress(80, 100, 'skipped archive file'));
|
|
156
121
|
}
|
|
157
|
-
// Check for cancellation before graph modification
|
|
158
|
-
checkCancellation(opts.signal, 'archiveEntities');
|
|
159
|
-
// Phase 3: Remove from main graph (80-100% progress)
|
|
160
|
-
reportProgress?.(createProgress(85, 100, 'updating graph'));
|
|
161
122
|
// Get mutable copy for write operation
|
|
162
123
|
const graph = await this.storage.getGraphForMutation();
|
|
163
124
|
// Remove archived entities from main graph
|
|
@@ -165,8 +126,6 @@ export class ArchiveManager {
|
|
|
165
126
|
graph.entities = graph.entities.filter(e => !archiveNames.has(e.name));
|
|
166
127
|
graph.relations = graph.relations.filter(r => !archiveNames.has(r.from) && !archiveNames.has(r.to));
|
|
167
128
|
await this.storage.saveGraph(graph);
|
|
168
|
-
// Report completion
|
|
169
|
-
reportProgress?.(createProgress(100, 100, 'archiveEntities'));
|
|
170
129
|
return {
|
|
171
130
|
archived: toArchive.length,
|
|
172
131
|
entityNames: toArchive.map(e => e.name),
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module features/CompressionManager
|
|
8
8
|
*/
|
|
9
|
-
import type { Entity, CompressionResult, KnowledgeGraph
|
|
9
|
+
import type { Entity, CompressionResult, KnowledgeGraph } from '../types/index.js';
|
|
10
10
|
import type { GraphStorage } from '../core/GraphStorage.js';
|
|
11
11
|
/**
|
|
12
12
|
* Manages compression operations for the knowledge graph.
|
|
@@ -18,10 +18,8 @@ export declare class CompressionManager {
|
|
|
18
18
|
* Prepare an entity for efficient similarity comparisons.
|
|
19
19
|
* Pre-computes all normalized data to avoid repeated computation.
|
|
20
20
|
*
|
|
21
|
-
* Phase 12 Sprint 1: Added nameHash for fast bucketing.
|
|
22
|
-
*
|
|
23
21
|
* @param entity - The entity to prepare
|
|
24
|
-
* @returns PreparedEntity with pre-computed data
|
|
22
|
+
* @returns PreparedEntity with pre-computed data
|
|
25
23
|
*/
|
|
26
24
|
private prepareEntity;
|
|
27
25
|
/**
|
|
@@ -67,16 +65,12 @@ export declare class CompressionManager {
|
|
|
67
65
|
* 2. Within each type, buckets by name prefix (first 2 chars normalized)
|
|
68
66
|
* 3. Only compares entities within same or adjacent buckets
|
|
69
67
|
*
|
|
70
|
-
* Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
|
|
71
|
-
*
|
|
72
68
|
* Complexity: O(n·k) where k is average bucket size (typically << n)
|
|
73
69
|
*
|
|
74
70
|
* @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
|
|
75
|
-
* @param options - Optional progress/cancellation options (Phase 9B)
|
|
76
71
|
* @returns Array of duplicate groups (each group has similar entities)
|
|
77
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
78
72
|
*/
|
|
79
|
-
findDuplicates(threshold?: number
|
|
73
|
+
findDuplicates(threshold?: number): Promise<string[][]>;
|
|
80
74
|
/**
|
|
81
75
|
* Merge a group of entities into a single entity.
|
|
82
76
|
*
|
|
@@ -106,14 +100,10 @@ export declare class CompressionManager {
|
|
|
106
100
|
* Compress the knowledge graph by finding and merging duplicates.
|
|
107
101
|
* OPTIMIZED: Loads graph once, performs all merges, saves once.
|
|
108
102
|
*
|
|
109
|
-
* Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
|
|
110
|
-
*
|
|
111
103
|
* @param threshold - Similarity threshold for duplicate detection (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
|
|
112
104
|
* @param dryRun - If true, only report what would be compressed without applying changes
|
|
113
|
-
* @param options - Optional progress/cancellation options (Phase 9B)
|
|
114
105
|
* @returns Compression result with statistics
|
|
115
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
116
106
|
*/
|
|
117
|
-
compressGraph(threshold?: number, dryRun?: boolean
|
|
107
|
+
compressGraph(threshold?: number, dryRun?: boolean): Promise<CompressionResult>;
|
|
118
108
|
}
|
|
119
109
|
//# sourceMappingURL=CompressionManager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CompressionManager.d.ts","sourceRoot":"","sources":["../../src/features/CompressionManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAY,iBAAiB,EAAE,cAAc,EAAE,
|
|
1
|
+
{"version":3,"file":"CompressionManager.d.ts","sourceRoot":"","sources":["../../src/features/CompressionManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAY,iBAAiB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAsB5D;;GAEG;AACH,qBAAa,kBAAkB;IACjB,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,YAAY;IAEzC;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAQvB;;;;;;;;;;;OAWG;IACH,yBAAyB,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM;IAwCzD;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IAoCnC;;;;;;;;;;;;OAYG;IACG,cAAc,CAAC,SAAS,GAAE,MAAoC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAiF1F;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EAAE,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE;QACP,KAAK,CAAC,EAAE,cAAc,CAAC;QACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACf,GACL,OAAO,CAAC,MAAM,CAAC;IA8FlB;;;;;;;OAOG;IACG,aAAa,CAAC,SAAS,GAAE,MAAoC,EAAE,MAAM,GAAE,OAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAoE1H"}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module features/CompressionManager
|
|
8
8
|
*/
|
|
9
|
-
import { levenshteinDistance
|
|
9
|
+
import { levenshteinDistance } from '../utils/index.js';
|
|
10
10
|
import { EntityNotFoundError, InsufficientEntitiesError } from '../utils/errors.js';
|
|
11
11
|
import { SIMILARITY_WEIGHTS, DEFAULT_DUPLICATE_THRESHOLD } from '../utils/constants.js';
|
|
12
12
|
/**
|
|
@@ -21,20 +21,16 @@ export class CompressionManager {
|
|
|
21
21
|
* Prepare an entity for efficient similarity comparisons.
|
|
22
22
|
* Pre-computes all normalized data to avoid repeated computation.
|
|
23
23
|
*
|
|
24
|
-
* Phase 12 Sprint 1: Added nameHash for fast bucketing.
|
|
25
|
-
*
|
|
26
24
|
* @param entity - The entity to prepare
|
|
27
|
-
* @returns PreparedEntity with pre-computed data
|
|
25
|
+
* @returns PreparedEntity with pre-computed data
|
|
28
26
|
*/
|
|
29
27
|
prepareEntity(entity) {
|
|
30
|
-
const nameLower = entity.name.toLowerCase();
|
|
31
28
|
return {
|
|
32
29
|
entity,
|
|
33
|
-
nameLower,
|
|
30
|
+
nameLower: entity.name.toLowerCase(),
|
|
34
31
|
typeLower: entity.entityType.toLowerCase(),
|
|
35
32
|
observationSet: new Set(entity.observations.map(o => o.toLowerCase())),
|
|
36
33
|
tagSet: new Set((entity.tags ?? []).map(t => t.toLowerCase())),
|
|
37
|
-
nameHash: fnv1aHash(nameLower),
|
|
38
34
|
};
|
|
39
35
|
}
|
|
40
36
|
/**
|
|
@@ -157,26 +153,15 @@ export class CompressionManager {
|
|
|
157
153
|
* 2. Within each type, buckets by name prefix (first 2 chars normalized)
|
|
158
154
|
* 3. Only compares entities within same or adjacent buckets
|
|
159
155
|
*
|
|
160
|
-
* Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
|
|
161
|
-
*
|
|
162
156
|
* Complexity: O(n·k) where k is average bucket size (typically << n)
|
|
163
157
|
*
|
|
164
158
|
* @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
|
|
165
|
-
* @param options - Optional progress/cancellation options (Phase 9B)
|
|
166
159
|
* @returns Array of duplicate groups (each group has similar entities)
|
|
167
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
168
160
|
*/
|
|
169
|
-
async findDuplicates(threshold = DEFAULT_DUPLICATE_THRESHOLD
|
|
170
|
-
// Check for early cancellation
|
|
171
|
-
checkCancellation(options?.signal, 'findDuplicates');
|
|
161
|
+
async findDuplicates(threshold = DEFAULT_DUPLICATE_THRESHOLD) {
|
|
172
162
|
const graph = await this.storage.loadGraph();
|
|
173
163
|
const duplicateGroups = [];
|
|
174
164
|
const processed = new Set();
|
|
175
|
-
// Setup progress reporter
|
|
176
|
-
const reportProgress = createProgressReporter(options?.onProgress);
|
|
177
|
-
const totalEntities = graph.entities.length;
|
|
178
|
-
let processedCount = 0;
|
|
179
|
-
reportProgress?.(createProgress(0, totalEntities, 'findDuplicates'));
|
|
180
165
|
// OPTIMIZATION: Pre-prepare all entities once before comparisons
|
|
181
166
|
const preparedEntities = this.prepareEntities(graph.entities);
|
|
182
167
|
// Step 1: Bucket entities by type (reduces comparisons drastically)
|
|
@@ -190,14 +175,9 @@ export class CompressionManager {
|
|
|
190
175
|
}
|
|
191
176
|
// Step 2: For each type bucket, sub-bucket by name prefix
|
|
192
177
|
for (const entities of typeMap.values()) {
|
|
193
|
-
// Check for cancellation between type buckets
|
|
194
|
-
checkCancellation(options?.signal, 'findDuplicates');
|
|
195
178
|
// Skip single-entity types (no duplicates possible)
|
|
196
|
-
if (entities.length < 2)
|
|
197
|
-
processedCount += entities.length;
|
|
198
|
-
reportProgress?.(createProgress(processedCount, totalEntities, 'findDuplicates'));
|
|
179
|
+
if (entities.length < 2)
|
|
199
180
|
continue;
|
|
200
|
-
}
|
|
201
181
|
// Create name prefix buckets (first 2 chars, normalized)
|
|
202
182
|
const prefixMap = new Map();
|
|
203
183
|
for (const entity of entities) {
|
|
@@ -210,8 +190,6 @@ export class CompressionManager {
|
|
|
210
190
|
// Step 3: Compare only within buckets (or adjacent buckets for fuzzy matching)
|
|
211
191
|
const prefixKeys = Array.from(prefixMap.keys()).sort();
|
|
212
192
|
for (let bucketIdx = 0; bucketIdx < prefixKeys.length; bucketIdx++) {
|
|
213
|
-
// Check for cancellation between prefix buckets
|
|
214
|
-
checkCancellation(options?.signal, 'findDuplicates');
|
|
215
193
|
const currentPrefix = prefixKeys[bucketIdx];
|
|
216
194
|
const currentBucket = prefixMap.get(currentPrefix);
|
|
217
195
|
// Collect entities to compare: current bucket + adjacent buckets
|
|
@@ -244,13 +222,9 @@ export class CompressionManager {
|
|
|
244
222
|
duplicateGroups.push(group);
|
|
245
223
|
processed.add(entity1.name);
|
|
246
224
|
}
|
|
247
|
-
processedCount++;
|
|
248
|
-
reportProgress?.(createProgress(processedCount, totalEntities, 'findDuplicates'));
|
|
249
225
|
}
|
|
250
226
|
}
|
|
251
227
|
}
|
|
252
|
-
// Report completion
|
|
253
|
-
reportProgress?.(createProgress(totalEntities, totalEntities, 'findDuplicates'));
|
|
254
228
|
return duplicateGroups;
|
|
255
229
|
}
|
|
256
230
|
/**
|
|
@@ -363,32 +337,12 @@ export class CompressionManager {
|
|
|
363
337
|
* Compress the knowledge graph by finding and merging duplicates.
|
|
364
338
|
* OPTIMIZED: Loads graph once, performs all merges, saves once.
|
|
365
339
|
*
|
|
366
|
-
* Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
|
|
367
|
-
*
|
|
368
340
|
* @param threshold - Similarity threshold for duplicate detection (0.0 to 1.0), default DEFAULT_DUPLICATE_THRESHOLD
|
|
369
341
|
* @param dryRun - If true, only report what would be compressed without applying changes
|
|
370
|
-
* @param options - Optional progress/cancellation options (Phase 9B)
|
|
371
342
|
* @returns Compression result with statistics
|
|
372
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
373
343
|
*/
|
|
374
|
-
async compressGraph(threshold = DEFAULT_DUPLICATE_THRESHOLD, dryRun = false
|
|
375
|
-
|
|
376
|
-
checkCancellation(options?.signal, 'compressGraph');
|
|
377
|
-
// Setup progress reporter (we'll use phases: 50% finding duplicates, 50% merging)
|
|
378
|
-
const reportProgress = createProgressReporter(options?.onProgress);
|
|
379
|
-
reportProgress?.(createProgress(0, 100, 'compressGraph'));
|
|
380
|
-
// Phase 1: Find duplicates (0-50% progress)
|
|
381
|
-
const duplicateGroups = await this.findDuplicates(threshold, {
|
|
382
|
-
signal: options?.signal,
|
|
383
|
-
onProgress: (p) => {
|
|
384
|
-
// Map findDuplicates progress (0-100%) to compressGraph progress (0-50%)
|
|
385
|
-
const compressProgress = Math.round(p.percentage * 0.5);
|
|
386
|
-
reportProgress?.(createProgress(compressProgress, 100, 'finding duplicates'));
|
|
387
|
-
},
|
|
388
|
-
});
|
|
389
|
-
// Check for cancellation after finding duplicates
|
|
390
|
-
checkCancellation(options?.signal, 'compressGraph');
|
|
391
|
-
reportProgress?.(createProgress(50, 100, 'compressGraph'));
|
|
344
|
+
async compressGraph(threshold = DEFAULT_DUPLICATE_THRESHOLD, dryRun = false) {
|
|
345
|
+
const duplicateGroups = await this.findDuplicates(threshold);
|
|
392
346
|
// OPTIMIZATION: Load graph once for all operations
|
|
393
347
|
const graph = await this.storage.getGraphForMutation();
|
|
394
348
|
const initialSize = JSON.stringify(graph).length;
|
|
@@ -409,26 +363,15 @@ export class CompressionManager {
|
|
|
409
363
|
});
|
|
410
364
|
result.entitiesMerged += group.length - 1;
|
|
411
365
|
}
|
|
412
|
-
reportProgress?.(createProgress(100, 100, 'compressGraph'));
|
|
413
366
|
return result;
|
|
414
367
|
}
|
|
415
|
-
// Phase 2: Merge duplicates (50-100% progress)
|
|
416
|
-
const totalGroups = duplicateGroups.length;
|
|
417
|
-
let mergedGroups = 0;
|
|
418
|
-
// OPTIMIZATION: Build entity lookup map for O(1) access during merges
|
|
419
|
-
const entityMap = new Map();
|
|
420
|
-
for (const entity of graph.entities) {
|
|
421
|
-
entityMap.set(entity.name, entity);
|
|
422
|
-
}
|
|
423
368
|
// Merge all duplicates using the same graph instance
|
|
424
369
|
for (const group of duplicateGroups) {
|
|
425
|
-
// Check for cancellation between merges
|
|
426
|
-
checkCancellation(options?.signal, 'compressGraph');
|
|
427
370
|
try {
|
|
428
|
-
// Count observations before merge using
|
|
371
|
+
// Count observations before merge using loaded graph
|
|
429
372
|
let totalObservationsBefore = 0;
|
|
430
373
|
for (const name of group) {
|
|
431
|
-
const entity =
|
|
374
|
+
const entity = graph.entities.find(e => e.name === name);
|
|
432
375
|
if (entity) {
|
|
433
376
|
totalObservationsBefore += entity.observations.length;
|
|
434
377
|
}
|
|
@@ -450,20 +393,12 @@ export class CompressionManager {
|
|
|
450
393
|
// Skip groups that fail to merge
|
|
451
394
|
console.error(`Failed to merge group ${group}:`, error);
|
|
452
395
|
}
|
|
453
|
-
mergedGroups++;
|
|
454
|
-
// Map merge progress (0-100%) to compressGraph progress (50-100%)
|
|
455
|
-
const mergeProgress = totalGroups > 0 ? Math.round(50 + (mergedGroups / totalGroups) * 50) : 100;
|
|
456
|
-
reportProgress?.(createProgress(mergeProgress, 100, 'merging entities'));
|
|
457
396
|
}
|
|
458
|
-
// Check for cancellation before final save
|
|
459
|
-
checkCancellation(options?.signal, 'compressGraph');
|
|
460
397
|
// OPTIMIZATION: Save once after all merges complete
|
|
461
398
|
await this.storage.saveGraph(graph);
|
|
462
399
|
const finalSize = JSON.stringify(graph).length;
|
|
463
400
|
result.spaceFreed = initialSize - finalSize;
|
|
464
401
|
result.relationsConsolidated = result.entitiesMerged;
|
|
465
|
-
// Report completion
|
|
466
|
-
reportProgress?.(createProgress(100, 100, 'compressGraph'));
|
|
467
402
|
return result;
|
|
468
403
|
}
|
|
469
404
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module features/IOManager
|
|
8
8
|
*/
|
|
9
|
-
import type { ReadonlyKnowledgeGraph, ImportResult, BackupOptions, BackupResult, RestoreResult, ExportOptions, ExportResult
|
|
9
|
+
import type { ReadonlyKnowledgeGraph, ImportResult, BackupOptions, BackupResult, RestoreResult, ExportOptions, ExportResult } from '../types/index.js';
|
|
10
10
|
import type { GraphStorage } from '../core/GraphStorage.js';
|
|
11
11
|
/**
|
|
12
12
|
* Supported export formats.
|
|
@@ -132,17 +132,13 @@ export declare class IOManager {
|
|
|
132
132
|
/**
|
|
133
133
|
* Import graph from formatted data.
|
|
134
134
|
*
|
|
135
|
-
* Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
|
|
136
|
-
*
|
|
137
135
|
* @param format - Import format
|
|
138
136
|
* @param data - Import data string
|
|
139
137
|
* @param mergeStrategy - How to handle conflicts
|
|
140
138
|
* @param dryRun - If true, preview changes without applying
|
|
141
|
-
* @param options - Optional progress/cancellation options (Phase 9B)
|
|
142
139
|
* @returns Import result with statistics
|
|
143
|
-
* @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
|
|
144
140
|
*/
|
|
145
|
-
importGraph(format: ImportFormat, data: string, mergeStrategy?: MergeStrategy, dryRun?: boolean
|
|
141
|
+
importGraph(format: ImportFormat, data: string, mergeStrategy?: MergeStrategy, dryRun?: boolean): Promise<ImportResult>;
|
|
146
142
|
private parseJsonImport;
|
|
147
143
|
private parseCsvImport;
|
|
148
144
|
private parseGraphMLImport;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IOManager.d.ts","sourceRoot":"","sources":["../../src/features/IOManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAIV,sBAAsB,EACtB,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EACb,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"IOManager.d.ts","sourceRoot":"","sources":["../../src/features/IOManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAIV,sBAAsB,EACtB,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EACb,aAAa,EACb,YAAY,EACb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAe5D;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,UAAU,GAAG,SAAS,CAAC;AAEhG;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAElE;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,8BAA8B;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,QAAQ,EAAE,cAAc,CAAC;IACzB,uCAAuC;IACvC,UAAU,EAAE,OAAO,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;CACd;AAMD;;;;;;;GAOG;AACH,qBAAa,SAAS;IAGR,OAAO,CAAC,OAAO;IAF3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEf,OAAO,EAAE,YAAY;IAUzC;;;;;;OAMG;IACH,WAAW,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM;IAqBxE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACG,0BAA0B,CAC9B,KAAK,EAAE,sBAAsB,EAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,YAAY,CAAC;IAuDxB;;;;;;;;;;;OAWG;YACW,YAAY;IA2C1B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,WAAW;IAoDnB,OAAO,CAAC,eAAe;IAuDvB,OAAO,CAAC,YAAY;IAqDpB,OAAO,CAAC,WAAW;IAiCnB,OAAO,CAAC,gBAAgB;IAwCxB,OAAO,CAAC,eAAe;IAyCvB;;;;;;;;OAQG;IACG,WAAW,CACf,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,aAAa,GAAE,aAAsB,EACrC,MAAM,GAAE,OAAe,GACtB,OAAO,CAAC,YAAY,CAAC;IA+BxB,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,cAAc;IAuGtB,OAAO,CAAC,kBAAkB;YAgEZ,kBAAkB;IA6GhC;;OAEG;YACW,eAAe;IAQ7B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAW9B;;;;;;;;;;;;;;;;;;OAkBG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiF3E;;;;;;OAMG;IACG,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IA+D1C;;;;;;;;;;;;;;;;;;OAkBG;IACG,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAoCnE;;;;OAIG;IACG,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrD;;;;;OAKG;IACG,eAAe,CAAC,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB9D;;OAEG;IACH,YAAY,IAAI,MAAM;CAGvB"}
|