@danielsimonjr/memory-mcp 12.2.3 → 12.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -102
- package/dist/index.js +6 -0
- package/dist/server/toolDefinitions.d.ts.map +1 -1
- package/dist/server/toolDefinitions.js +745 -3
- package/dist/server/toolHandlers.d.ts +1 -9
- package/dist/server/toolHandlers.d.ts.map +1 -1
- package/dist/server/toolHandlers.js +859 -11
- package/package.json +9 -6
- package/dist/core/EntityManager.d.ts +0 -263
- package/dist/core/EntityManager.d.ts.map +0 -1
- package/dist/core/EntityManager.js +0 -478
- package/dist/core/GraphStorage.d.ts +0 -344
- package/dist/core/GraphStorage.d.ts.map +0 -1
- package/dist/core/GraphStorage.js +0 -708
- package/dist/core/GraphTraversal.d.ts +0 -136
- package/dist/core/GraphTraversal.d.ts.map +0 -1
- package/dist/core/GraphTraversal.js +0 -556
- package/dist/core/HierarchyManager.d.ts +0 -111
- package/dist/core/HierarchyManager.d.ts.map +0 -1
- package/dist/core/HierarchyManager.js +0 -224
- package/dist/core/ManagerContext.d.ts +0 -72
- package/dist/core/ManagerContext.d.ts.map +0 -1
- package/dist/core/ManagerContext.js +0 -118
- package/dist/core/ObservationManager.d.ts +0 -85
- package/dist/core/ObservationManager.d.ts.map +0 -1
- package/dist/core/ObservationManager.js +0 -123
- package/dist/core/RelationManager.d.ts +0 -131
- package/dist/core/RelationManager.d.ts.map +0 -1
- package/dist/core/RelationManager.js +0 -210
- package/dist/core/SQLiteStorage.d.ts +0 -354
- package/dist/core/SQLiteStorage.d.ts.map +0 -1
- package/dist/core/SQLiteStorage.js +0 -917
- package/dist/core/StorageFactory.d.ts +0 -45
- package/dist/core/StorageFactory.d.ts.map +0 -1
- package/dist/core/StorageFactory.js +0 -64
- package/dist/core/TransactionManager.d.ts +0 -259
- package/dist/core/TransactionManager.d.ts.map +0 -1
- package/dist/core/TransactionManager.js +0 -392
- package/dist/core/index.d.ts +0 -16
- package/dist/core/index.d.ts.map +0 -1
- package/dist/core/index.js +0 -17
- package/dist/features/AnalyticsManager.d.ts +0 -44
- package/dist/features/AnalyticsManager.d.ts.map +0 -1
- package/dist/features/AnalyticsManager.js +0 -223
- package/dist/features/ArchiveManager.d.ts +0 -121
- package/dist/features/ArchiveManager.d.ts.map +0 -1
- package/dist/features/ArchiveManager.js +0 -240
- package/dist/features/CompressionManager.d.ts +0 -109
- package/dist/features/CompressionManager.d.ts.map +0 -1
- package/dist/features/CompressionManager.js +0 -404
- package/dist/features/IOManager.d.ts +0 -221
- package/dist/features/IOManager.d.ts.map +0 -1
- package/dist/features/IOManager.js +0 -997
- package/dist/features/StreamingExporter.d.ts +0 -105
- package/dist/features/StreamingExporter.d.ts.map +0 -1
- package/dist/features/StreamingExporter.js +0 -150
- package/dist/features/TagManager.d.ts +0 -147
- package/dist/features/TagManager.d.ts.map +0 -1
- package/dist/features/TagManager.js +0 -210
- package/dist/features/index.d.ts +0 -12
- package/dist/features/index.d.ts.map +0 -1
- package/dist/features/index.js +0 -11
- package/dist/search/BasicSearch.d.ts +0 -51
- package/dist/search/BasicSearch.d.ts.map +0 -1
- package/dist/search/BasicSearch.js +0 -137
- package/dist/search/BooleanSearch.d.ts +0 -98
- package/dist/search/BooleanSearch.d.ts.map +0 -1
- package/dist/search/BooleanSearch.js +0 -430
- package/dist/search/EmbeddingService.d.ts +0 -178
- package/dist/search/EmbeddingService.d.ts.map +0 -1
- package/dist/search/EmbeddingService.js +0 -358
- package/dist/search/FuzzySearch.d.ts +0 -118
- package/dist/search/FuzzySearch.d.ts.map +0 -1
- package/dist/search/FuzzySearch.js +0 -312
- package/dist/search/RankedSearch.d.ts +0 -71
- package/dist/search/RankedSearch.d.ts.map +0 -1
- package/dist/search/RankedSearch.js +0 -238
- package/dist/search/SavedSearchManager.d.ts +0 -79
- package/dist/search/SavedSearchManager.d.ts.map +0 -1
- package/dist/search/SavedSearchManager.js +0 -145
- package/dist/search/SearchFilterChain.d.ts +0 -120
- package/dist/search/SearchFilterChain.d.ts.map +0 -1
- package/dist/search/SearchFilterChain.js +0 -185
- package/dist/search/SearchManager.d.ts +0 -285
- package/dist/search/SearchManager.d.ts.map +0 -1
- package/dist/search/SearchManager.js +0 -338
- package/dist/search/SearchSuggestions.d.ts +0 -27
- package/dist/search/SearchSuggestions.d.ts.map +0 -1
- package/dist/search/SearchSuggestions.js +0 -57
- package/dist/search/SemanticSearch.d.ts +0 -146
- package/dist/search/SemanticSearch.d.ts.map +0 -1
- package/dist/search/SemanticSearch.js +0 -313
- package/dist/search/TFIDFIndexManager.d.ts +0 -63
- package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
- package/dist/search/TFIDFIndexManager.js +0 -215
- package/dist/search/VectorStore.d.ts +0 -235
- package/dist/search/VectorStore.d.ts.map +0 -1
- package/dist/search/VectorStore.js +0 -311
- package/dist/search/index.d.ts +0 -18
- package/dist/search/index.d.ts.map +0 -1
- package/dist/search/index.js +0 -21
- package/dist/types/index.d.ts +0 -13
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -12
- package/dist/types/types.d.ts +0 -1234
- package/dist/types/types.d.ts.map +0 -1
- package/dist/types/types.js +0 -9
- package/dist/utils/compressedCache.d.ts +0 -192
- package/dist/utils/compressedCache.d.ts.map +0 -1
- package/dist/utils/compressedCache.js +0 -309
- package/dist/utils/compressionUtil.d.ts +0 -214
- package/dist/utils/compressionUtil.d.ts.map +0 -1
- package/dist/utils/compressionUtil.js +0 -247
- package/dist/utils/constants.d.ts +0 -245
- package/dist/utils/constants.d.ts.map +0 -1
- package/dist/utils/constants.js +0 -252
- package/dist/utils/entityUtils.d.ts +0 -321
- package/dist/utils/entityUtils.d.ts.map +0 -1
- package/dist/utils/entityUtils.js +0 -538
- package/dist/utils/errors.d.ts +0 -77
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -121
- package/dist/utils/formatters.d.ts +0 -145
- package/dist/utils/formatters.d.ts.map +0 -1
- package/dist/utils/formatters.js +0 -132
- package/dist/utils/index.d.ts +0 -22
- package/dist/utils/index.d.ts.map +0 -1
- package/dist/utils/index.js +0 -75
- package/dist/utils/indexes.d.ts +0 -270
- package/dist/utils/indexes.d.ts.map +0 -1
- package/dist/utils/indexes.js +0 -526
- package/dist/utils/logger.d.ts +0 -24
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -33
- package/dist/utils/parallelUtils.d.ts +0 -72
- package/dist/utils/parallelUtils.d.ts.map +0 -1
- package/dist/utils/parallelUtils.js +0 -169
- package/dist/utils/schemas.d.ts +0 -374
- package/dist/utils/schemas.d.ts.map +0 -1
- package/dist/utils/schemas.js +0 -484
- package/dist/utils/searchAlgorithms.d.ts +0 -99
- package/dist/utils/searchAlgorithms.d.ts.map +0 -1
- package/dist/utils/searchAlgorithms.js +0 -167
- package/dist/utils/searchCache.d.ts +0 -108
- package/dist/utils/searchCache.d.ts.map +0 -1
- package/dist/utils/searchCache.js +0 -209
- package/dist/utils/taskScheduler.d.ts +0 -290
- package/dist/utils/taskScheduler.d.ts.map +0 -1
- package/dist/utils/taskScheduler.js +0 -466
- package/dist/workers/WorkerPool.d.ts +0 -81
- package/dist/workers/WorkerPool.d.ts.map +0 -1
- package/dist/workers/WorkerPool.js +0 -121
- package/dist/workers/index.d.ts +0 -12
- package/dist/workers/index.d.ts.map +0 -1
- package/dist/workers/index.js +0 -9
- package/dist/workers/levenshteinWorker.d.ts +0 -60
- package/dist/workers/levenshteinWorker.d.ts.map +0 -1
- package/dist/workers/levenshteinWorker.js +0 -98
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Analytics Manager
|
|
3
|
-
*
|
|
4
|
-
* Handles graph statistics and validation operations.
|
|
5
|
-
* Extracted from SearchManager (Phase 4: Consolidate God Objects).
|
|
6
|
-
*
|
|
7
|
-
* @module features/AnalyticsManager
|
|
8
|
-
*/
|
|
9
|
-
/**
|
|
10
|
-
* Manages analytics operations for the knowledge graph.
|
|
11
|
-
*/
|
|
12
|
-
export class AnalyticsManager {
|
|
13
|
-
storage;
|
|
14
|
-
constructor(storage) {
|
|
15
|
-
this.storage = storage;
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* Validate the knowledge graph structure and data integrity.
|
|
19
|
-
*
|
|
20
|
-
* Checks for:
|
|
21
|
-
* - Orphaned relations (pointing to non-existent entities)
|
|
22
|
-
* - Duplicate entity names
|
|
23
|
-
* - Invalid entity data (missing name/type, invalid observations)
|
|
24
|
-
* - Isolated entities (no relations)
|
|
25
|
-
* - Empty observations
|
|
26
|
-
* - Missing metadata (createdAt, lastModified)
|
|
27
|
-
*
|
|
28
|
-
* @returns Validation report with errors, warnings, and summary
|
|
29
|
-
*/
|
|
30
|
-
async validateGraph() {
|
|
31
|
-
const graph = await this.storage.loadGraph();
|
|
32
|
-
const issues = [];
|
|
33
|
-
const warnings = [];
|
|
34
|
-
// Create a set of all entity names for fast lookup
|
|
35
|
-
const entityNames = new Set(graph.entities.map(e => e.name));
|
|
36
|
-
// Check for orphaned relations (relations pointing to non-existent entities)
|
|
37
|
-
for (const relation of graph.relations) {
|
|
38
|
-
if (!entityNames.has(relation.from)) {
|
|
39
|
-
issues.push({
|
|
40
|
-
type: 'orphaned_relation',
|
|
41
|
-
message: `Relation has non-existent source entity: "${relation.from}"`,
|
|
42
|
-
details: { relation, missingEntity: relation.from },
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
if (!entityNames.has(relation.to)) {
|
|
46
|
-
issues.push({
|
|
47
|
-
type: 'orphaned_relation',
|
|
48
|
-
message: `Relation has non-existent target entity: "${relation.to}"`,
|
|
49
|
-
details: { relation, missingEntity: relation.to },
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
// Check for duplicate entity names
|
|
54
|
-
const entityNameCounts = new Map();
|
|
55
|
-
for (const entity of graph.entities) {
|
|
56
|
-
const count = entityNameCounts.get(entity.name) || 0;
|
|
57
|
-
entityNameCounts.set(entity.name, count + 1);
|
|
58
|
-
}
|
|
59
|
-
for (const [name, count] of entityNameCounts.entries()) {
|
|
60
|
-
if (count > 1) {
|
|
61
|
-
issues.push({
|
|
62
|
-
type: 'duplicate_entity',
|
|
63
|
-
message: `Duplicate entity name found: "${name}" (${count} instances)`,
|
|
64
|
-
details: { entityName: name, count },
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
// Check for entities with invalid data
|
|
69
|
-
for (const entity of graph.entities) {
|
|
70
|
-
if (!entity.name || entity.name.trim() === '') {
|
|
71
|
-
issues.push({
|
|
72
|
-
type: 'invalid_data',
|
|
73
|
-
message: 'Entity has empty or missing name',
|
|
74
|
-
details: { entity },
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
if (!entity.entityType || entity.entityType.trim() === '') {
|
|
78
|
-
issues.push({
|
|
79
|
-
type: 'invalid_data',
|
|
80
|
-
message: `Entity "${entity.name}" has empty or missing entityType`,
|
|
81
|
-
details: { entity },
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
if (!Array.isArray(entity.observations)) {
|
|
85
|
-
issues.push({
|
|
86
|
-
type: 'invalid_data',
|
|
87
|
-
message: `Entity "${entity.name}" has invalid observations (not an array)`,
|
|
88
|
-
details: { entity },
|
|
89
|
-
});
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
// Warnings: Check for isolated entities (no relations)
|
|
93
|
-
const entitiesInRelations = new Set();
|
|
94
|
-
for (const relation of graph.relations) {
|
|
95
|
-
entitiesInRelations.add(relation.from);
|
|
96
|
-
entitiesInRelations.add(relation.to);
|
|
97
|
-
}
|
|
98
|
-
for (const entity of graph.entities) {
|
|
99
|
-
if (!entitiesInRelations.has(entity.name) && graph.relations.length > 0) {
|
|
100
|
-
warnings.push({
|
|
101
|
-
type: 'isolated_entity',
|
|
102
|
-
message: `Entity "${entity.name}" has no relations to other entities`,
|
|
103
|
-
details: { entityName: entity.name },
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
// Warnings: Check for entities with empty observations
|
|
108
|
-
for (const entity of graph.entities) {
|
|
109
|
-
if (entity.observations.length === 0) {
|
|
110
|
-
warnings.push({
|
|
111
|
-
type: 'empty_observations',
|
|
112
|
-
message: `Entity "${entity.name}" has no observations`,
|
|
113
|
-
details: { entityName: entity.name },
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
// Warnings: Check for missing metadata (createdAt, lastModified)
|
|
118
|
-
for (const entity of graph.entities) {
|
|
119
|
-
if (!entity.createdAt) {
|
|
120
|
-
warnings.push({
|
|
121
|
-
type: 'missing_metadata',
|
|
122
|
-
message: `Entity "${entity.name}" is missing createdAt timestamp`,
|
|
123
|
-
details: { entityName: entity.name, field: 'createdAt' },
|
|
124
|
-
});
|
|
125
|
-
}
|
|
126
|
-
if (!entity.lastModified) {
|
|
127
|
-
warnings.push({
|
|
128
|
-
type: 'missing_metadata',
|
|
129
|
-
message: `Entity "${entity.name}" is missing lastModified timestamp`,
|
|
130
|
-
details: { entityName: entity.name, field: 'lastModified' },
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
// Count specific issues
|
|
135
|
-
const orphanedRelationsCount = issues.filter(e => e.type === 'orphaned_relation').length;
|
|
136
|
-
const entitiesWithoutRelationsCount = warnings.filter(w => w.type === 'isolated_entity').length;
|
|
137
|
-
return {
|
|
138
|
-
isValid: issues.length === 0,
|
|
139
|
-
issues,
|
|
140
|
-
warnings,
|
|
141
|
-
summary: {
|
|
142
|
-
totalErrors: issues.length,
|
|
143
|
-
totalWarnings: warnings.length,
|
|
144
|
-
orphanedRelationsCount,
|
|
145
|
-
entitiesWithoutRelationsCount,
|
|
146
|
-
},
|
|
147
|
-
};
|
|
148
|
-
}
|
|
149
|
-
/**
|
|
150
|
-
* Get comprehensive statistics about the knowledge graph.
|
|
151
|
-
*
|
|
152
|
-
* Provides metrics including:
|
|
153
|
-
* - Total counts of entities and relations
|
|
154
|
-
* - Entity and relation type distributions
|
|
155
|
-
* - Oldest and newest entities/relations
|
|
156
|
-
* - Date ranges for entities and relations
|
|
157
|
-
*
|
|
158
|
-
* @returns Graph statistics object
|
|
159
|
-
*/
|
|
160
|
-
async getGraphStats() {
|
|
161
|
-
const graph = await this.storage.loadGraph();
|
|
162
|
-
// Calculate entity type counts
|
|
163
|
-
const entityTypesCounts = {};
|
|
164
|
-
graph.entities.forEach(e => {
|
|
165
|
-
entityTypesCounts[e.entityType] = (entityTypesCounts[e.entityType] || 0) + 1;
|
|
166
|
-
});
|
|
167
|
-
// Calculate relation type counts
|
|
168
|
-
const relationTypesCounts = {};
|
|
169
|
-
graph.relations.forEach(r => {
|
|
170
|
-
relationTypesCounts[r.relationType] = (relationTypesCounts[r.relationType] || 0) + 1;
|
|
171
|
-
});
|
|
172
|
-
// Find oldest and newest entities
|
|
173
|
-
let oldestEntity;
|
|
174
|
-
let newestEntity;
|
|
175
|
-
let earliestEntityDate = null;
|
|
176
|
-
let latestEntityDate = null;
|
|
177
|
-
graph.entities.forEach(e => {
|
|
178
|
-
const date = new Date(e.createdAt || '');
|
|
179
|
-
if (!earliestEntityDate || date < earliestEntityDate) {
|
|
180
|
-
earliestEntityDate = date;
|
|
181
|
-
oldestEntity = { name: e.name, date: e.createdAt || '' };
|
|
182
|
-
}
|
|
183
|
-
if (!latestEntityDate || date > latestEntityDate) {
|
|
184
|
-
latestEntityDate = date;
|
|
185
|
-
newestEntity = { name: e.name, date: e.createdAt || '' };
|
|
186
|
-
}
|
|
187
|
-
});
|
|
188
|
-
// Find oldest and newest relations
|
|
189
|
-
let oldestRelation;
|
|
190
|
-
let newestRelation;
|
|
191
|
-
let earliestRelationDate = null;
|
|
192
|
-
let latestRelationDate = null;
|
|
193
|
-
graph.relations.forEach(r => {
|
|
194
|
-
const date = new Date(r.createdAt || '');
|
|
195
|
-
if (!earliestRelationDate || date < earliestRelationDate) {
|
|
196
|
-
earliestRelationDate = date;
|
|
197
|
-
oldestRelation = { from: r.from, to: r.to, relationType: r.relationType, date: r.createdAt || '' };
|
|
198
|
-
}
|
|
199
|
-
if (!latestRelationDate || date > latestRelationDate) {
|
|
200
|
-
latestRelationDate = date;
|
|
201
|
-
newestRelation = { from: r.from, to: r.to, relationType: r.relationType, date: r.createdAt || '' };
|
|
202
|
-
}
|
|
203
|
-
});
|
|
204
|
-
return {
|
|
205
|
-
totalEntities: graph.entities.length,
|
|
206
|
-
totalRelations: graph.relations.length,
|
|
207
|
-
entityTypesCounts,
|
|
208
|
-
relationTypesCounts,
|
|
209
|
-
oldestEntity,
|
|
210
|
-
newestEntity,
|
|
211
|
-
oldestRelation,
|
|
212
|
-
newestRelation,
|
|
213
|
-
entityDateRange: earliestEntityDate && latestEntityDate ? {
|
|
214
|
-
earliest: earliestEntityDate.toISOString(),
|
|
215
|
-
latest: latestEntityDate.toISOString()
|
|
216
|
-
} : undefined,
|
|
217
|
-
relationDateRange: earliestRelationDate && latestRelationDate ? {
|
|
218
|
-
earliest: earliestRelationDate.toISOString(),
|
|
219
|
-
latest: latestRelationDate.toISOString()
|
|
220
|
-
} : undefined,
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
}
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Archive Manager
|
|
3
|
-
*
|
|
4
|
-
* Handles archiving (removal) of entities based on criteria.
|
|
5
|
-
* Archives are stored as compressed files for space-efficient long-term storage.
|
|
6
|
-
* Extracted from EntityManager (Phase 4: Consolidate God Objects).
|
|
7
|
-
* Enhanced with brotli compression in Phase 3 Sprint 5.
|
|
8
|
-
*
|
|
9
|
-
* @module features/ArchiveManager
|
|
10
|
-
*/
|
|
11
|
-
import type { GraphStorage } from '../core/GraphStorage.js';
|
|
12
|
-
/**
|
|
13
|
-
* Criteria for archiving entities.
|
|
14
|
-
*/
|
|
15
|
-
export interface ArchiveCriteria {
|
|
16
|
-
/** Entities older than this date (ISO 8601) */
|
|
17
|
-
olderThan?: string;
|
|
18
|
-
/** Entities with importance less than this value */
|
|
19
|
-
importanceLessThan?: number;
|
|
20
|
-
/** Entities with any of these tags */
|
|
21
|
-
tags?: string[];
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Options for archive operations.
|
|
25
|
-
*/
|
|
26
|
-
export interface ArchiveOptions {
|
|
27
|
-
/** Dry run mode - preview without making changes */
|
|
28
|
-
dryRun?: boolean;
|
|
29
|
-
/** Whether to save archived entities to a compressed file (default: true) */
|
|
30
|
-
saveToFile?: boolean;
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Result of archive operation.
|
|
34
|
-
* Extends ArchiveResultExtended with compression statistics.
|
|
35
|
-
*/
|
|
36
|
-
export interface ArchiveResult {
|
|
37
|
-
/** Number of entities archived */
|
|
38
|
-
archived: number;
|
|
39
|
-
/** Names of archived entities */
|
|
40
|
-
entityNames: string[];
|
|
41
|
-
/** Path to the archive file (if created) */
|
|
42
|
-
archivePath?: string;
|
|
43
|
-
/** Original size of archive data in bytes */
|
|
44
|
-
originalSize?: number;
|
|
45
|
-
/** Compressed size in bytes */
|
|
46
|
-
compressedSize?: number;
|
|
47
|
-
/** Compression ratio (compressedSize / originalSize). Lower is better. */
|
|
48
|
-
compressionRatio?: number;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Manages archive operations for the knowledge graph.
|
|
52
|
-
*
|
|
53
|
-
* Archives are stored as brotli-compressed files in the `.archives` directory.
|
|
54
|
-
* Maximum compression quality is used for optimal long-term storage.
|
|
55
|
-
*/
|
|
56
|
-
export declare class ArchiveManager {
|
|
57
|
-
private storage;
|
|
58
|
-
private readonly archiveDir;
|
|
59
|
-
constructor(storage: GraphStorage);
|
|
60
|
-
/**
|
|
61
|
-
* Archive old or low-importance entities.
|
|
62
|
-
*
|
|
63
|
-
* Entities matching ANY of the criteria are archived:
|
|
64
|
-
* - lastModified older than olderThan date
|
|
65
|
-
* - importance less than importanceLessThan
|
|
66
|
-
* - has at least one tag from tags array
|
|
67
|
-
*
|
|
68
|
-
* By default, archived entities are saved to a compressed file before
|
|
69
|
-
* being removed from the active graph. Use `saveToFile: false` to
|
|
70
|
-
* skip creating the archive file.
|
|
71
|
-
*
|
|
72
|
-
* @param criteria - Archiving criteria
|
|
73
|
-
* @param options - Archive options (dryRun, saveToFile)
|
|
74
|
-
* @returns Archive result with count, entity names, and compression stats
|
|
75
|
-
*
|
|
76
|
-
* @example
|
|
77
|
-
* ```typescript
|
|
78
|
-
* // Archive old entities with compression
|
|
79
|
-
* const result = await manager.archiveEntities({
|
|
80
|
-
* olderThan: '2023-01-01T00:00:00Z',
|
|
81
|
-
* importanceLessThan: 3
|
|
82
|
-
* });
|
|
83
|
-
* console.log(`Archived ${result.archived} entities`);
|
|
84
|
-
* console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
|
|
85
|
-
*
|
|
86
|
-
* // Preview without making changes
|
|
87
|
-
* const preview = await manager.archiveEntities(criteria, { dryRun: true });
|
|
88
|
-
* ```
|
|
89
|
-
*/
|
|
90
|
-
archiveEntities(criteria: ArchiveCriteria, options?: ArchiveOptions | boolean): Promise<ArchiveResult>;
|
|
91
|
-
/**
|
|
92
|
-
* Save entities to a compressed archive file.
|
|
93
|
-
*
|
|
94
|
-
* Creates a brotli-compressed file in the `.archives` directory
|
|
95
|
-
* with maximum compression quality for space efficiency.
|
|
96
|
-
*
|
|
97
|
-
* @param entities - Entities to archive
|
|
98
|
-
* @returns Archive file path and compression statistics
|
|
99
|
-
*/
|
|
100
|
-
private saveToArchive;
|
|
101
|
-
/**
|
|
102
|
-
* List all available archives.
|
|
103
|
-
*
|
|
104
|
-
* @returns Array of archive information with compression details
|
|
105
|
-
*/
|
|
106
|
-
listArchives(): Promise<Array<{
|
|
107
|
-
fileName: string;
|
|
108
|
-
filePath: string;
|
|
109
|
-
timestamp: string;
|
|
110
|
-
entityCount: number;
|
|
111
|
-
compressed: boolean;
|
|
112
|
-
originalSize?: number;
|
|
113
|
-
compressedSize?: number;
|
|
114
|
-
compressionRatio?: number;
|
|
115
|
-
}>>;
|
|
116
|
-
/**
|
|
117
|
-
* Get the path to the archives directory.
|
|
118
|
-
*/
|
|
119
|
-
getArchiveDir(): string;
|
|
120
|
-
}
|
|
121
|
-
//# sourceMappingURL=ArchiveManager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ArchiveManager.d.ts","sourceRoot":"","sources":["../../src/features/ArchiveManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oDAAoD;IACpD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+BAA+B;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,qBAAa,cAAc;IAGb,OAAO,CAAC,OAAO;IAF3B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAEhB,OAAO,EAAE,YAAY;IAMzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,eAAe,CACnB,QAAQ,EAAE,eAAe,EACzB,OAAO,GAAE,cAAc,GAAG,OAAY,GACrC,OAAO,CAAC,aAAa,CAAC;IA+FzB;;;;;;;;OAQG;YACW,aAAa;IAmD3B;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC;QAClC,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,OAAO,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IA6DH;;OAEG;IACH,aAAa,IAAI,MAAM;CAGxB"}
|
|
@@ -1,240 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Archive Manager
|
|
3
|
-
*
|
|
4
|
-
* Handles archiving (removal) of entities based on criteria.
|
|
5
|
-
* Archives are stored as compressed files for space-efficient long-term storage.
|
|
6
|
-
* Extracted from EntityManager (Phase 4: Consolidate God Objects).
|
|
7
|
-
* Enhanced with brotli compression in Phase 3 Sprint 5.
|
|
8
|
-
*
|
|
9
|
-
* @module features/ArchiveManager
|
|
10
|
-
*/
|
|
11
|
-
import { promises as fs } from 'fs';
|
|
12
|
-
import { dirname, join } from 'path';
|
|
13
|
-
import { compress, COMPRESSION_CONFIG } from '../utils/index.js';
|
|
14
|
-
/**
|
|
15
|
-
* Manages archive operations for the knowledge graph.
|
|
16
|
-
*
|
|
17
|
-
* Archives are stored as brotli-compressed files in the `.archives` directory.
|
|
18
|
-
* Maximum compression quality is used for optimal long-term storage.
|
|
19
|
-
*/
|
|
20
|
-
export class ArchiveManager {
|
|
21
|
-
storage;
|
|
22
|
-
archiveDir;
|
|
23
|
-
constructor(storage) {
|
|
24
|
-
this.storage = storage;
|
|
25
|
-
const filePath = this.storage.getFilePath();
|
|
26
|
-
const dir = dirname(filePath);
|
|
27
|
-
this.archiveDir = join(dir, '.archives');
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Archive old or low-importance entities.
|
|
31
|
-
*
|
|
32
|
-
* Entities matching ANY of the criteria are archived:
|
|
33
|
-
* - lastModified older than olderThan date
|
|
34
|
-
* - importance less than importanceLessThan
|
|
35
|
-
* - has at least one tag from tags array
|
|
36
|
-
*
|
|
37
|
-
* By default, archived entities are saved to a compressed file before
|
|
38
|
-
* being removed from the active graph. Use `saveToFile: false` to
|
|
39
|
-
* skip creating the archive file.
|
|
40
|
-
*
|
|
41
|
-
* @param criteria - Archiving criteria
|
|
42
|
-
* @param options - Archive options (dryRun, saveToFile)
|
|
43
|
-
* @returns Archive result with count, entity names, and compression stats
|
|
44
|
-
*
|
|
45
|
-
* @example
|
|
46
|
-
* ```typescript
|
|
47
|
-
* // Archive old entities with compression
|
|
48
|
-
* const result = await manager.archiveEntities({
|
|
49
|
-
* olderThan: '2023-01-01T00:00:00Z',
|
|
50
|
-
* importanceLessThan: 3
|
|
51
|
-
* });
|
|
52
|
-
* console.log(`Archived ${result.archived} entities`);
|
|
53
|
-
* console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
|
|
54
|
-
*
|
|
55
|
-
* // Preview without making changes
|
|
56
|
-
* const preview = await manager.archiveEntities(criteria, { dryRun: true });
|
|
57
|
-
* ```
|
|
58
|
-
*/
|
|
59
|
-
async archiveEntities(criteria, options = {}) {
|
|
60
|
-
// Handle legacy boolean argument (backward compatibility)
|
|
61
|
-
const opts = typeof options === 'boolean'
|
|
62
|
-
? { dryRun: options, saveToFile: true }
|
|
63
|
-
: { saveToFile: true, ...options };
|
|
64
|
-
// Use read-only graph for analysis
|
|
65
|
-
const readGraph = await this.storage.loadGraph();
|
|
66
|
-
const toArchive = [];
|
|
67
|
-
for (const entity of readGraph.entities) {
|
|
68
|
-
let shouldArchive = false;
|
|
69
|
-
// Check age criteria
|
|
70
|
-
if (criteria.olderThan && entity.lastModified) {
|
|
71
|
-
const entityDate = new Date(entity.lastModified);
|
|
72
|
-
const cutoffDate = new Date(criteria.olderThan);
|
|
73
|
-
if (entityDate < cutoffDate) {
|
|
74
|
-
shouldArchive = true;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
// Check importance criteria
|
|
78
|
-
if (criteria.importanceLessThan !== undefined) {
|
|
79
|
-
if (entity.importance === undefined || entity.importance < criteria.importanceLessThan) {
|
|
80
|
-
shouldArchive = true;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
// Check tag criteria (must have at least one matching tag)
|
|
84
|
-
if (criteria.tags && criteria.tags.length > 0) {
|
|
85
|
-
const normalizedCriteriaTags = criteria.tags.map(t => t.toLowerCase());
|
|
86
|
-
const entityTags = (entity.tags || []).map(t => t.toLowerCase());
|
|
87
|
-
const hasMatchingTag = normalizedCriteriaTags.some(tag => entityTags.includes(tag));
|
|
88
|
-
if (hasMatchingTag) {
|
|
89
|
-
shouldArchive = true;
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (shouldArchive) {
|
|
93
|
-
toArchive.push(entity);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
// Dry run - return preview without changes
|
|
97
|
-
if (opts.dryRun) {
|
|
98
|
-
return {
|
|
99
|
-
archived: toArchive.length,
|
|
100
|
-
entityNames: toArchive.map(e => e.name),
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
// No entities to archive
|
|
104
|
-
if (toArchive.length === 0) {
|
|
105
|
-
return {
|
|
106
|
-
archived: 0,
|
|
107
|
-
entityNames: [],
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
// Save to compressed archive file
|
|
111
|
-
let archivePath;
|
|
112
|
-
let originalSize;
|
|
113
|
-
let compressedSize;
|
|
114
|
-
let compressionRatio;
|
|
115
|
-
if (opts.saveToFile) {
|
|
116
|
-
const archiveResult = await this.saveToArchive(toArchive);
|
|
117
|
-
archivePath = archiveResult.archivePath;
|
|
118
|
-
originalSize = archiveResult.originalSize;
|
|
119
|
-
compressedSize = archiveResult.compressedSize;
|
|
120
|
-
compressionRatio = archiveResult.compressionRatio;
|
|
121
|
-
}
|
|
122
|
-
// Get mutable copy for write operation
|
|
123
|
-
const graph = await this.storage.getGraphForMutation();
|
|
124
|
-
// Remove archived entities from main graph
|
|
125
|
-
const archiveNames = new Set(toArchive.map(e => e.name));
|
|
126
|
-
graph.entities = graph.entities.filter(e => !archiveNames.has(e.name));
|
|
127
|
-
graph.relations = graph.relations.filter(r => !archiveNames.has(r.from) && !archiveNames.has(r.to));
|
|
128
|
-
await this.storage.saveGraph(graph);
|
|
129
|
-
return {
|
|
130
|
-
archived: toArchive.length,
|
|
131
|
-
entityNames: toArchive.map(e => e.name),
|
|
132
|
-
archivePath,
|
|
133
|
-
originalSize,
|
|
134
|
-
compressedSize,
|
|
135
|
-
compressionRatio,
|
|
136
|
-
};
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Save entities to a compressed archive file.
|
|
140
|
-
*
|
|
141
|
-
* Creates a brotli-compressed file in the `.archives` directory
|
|
142
|
-
* with maximum compression quality for space efficiency.
|
|
143
|
-
*
|
|
144
|
-
* @param entities - Entities to archive
|
|
145
|
-
* @returns Archive file path and compression statistics
|
|
146
|
-
*/
|
|
147
|
-
async saveToArchive(entities) {
|
|
148
|
-
// Ensure archive directory exists
|
|
149
|
-
await fs.mkdir(this.archiveDir, { recursive: true });
|
|
150
|
-
// Generate timestamp-based filename
|
|
151
|
-
const timestamp = new Date().toISOString()
|
|
152
|
-
.replace(/:/g, '-')
|
|
153
|
-
.replace(/\./g, '-')
|
|
154
|
-
.replace('T', '_')
|
|
155
|
-
.replace('Z', '');
|
|
156
|
-
const archivePath = join(this.archiveDir, `archive_${timestamp}.jsonl.br`);
|
|
157
|
-
// Serialize entities to JSONL format
|
|
158
|
-
const content = entities.map(e => JSON.stringify(e)).join('\n');
|
|
159
|
-
// Compress with maximum quality for archives
|
|
160
|
-
const compressionResult = await compress(content, {
|
|
161
|
-
quality: COMPRESSION_CONFIG.BROTLI_QUALITY_ARCHIVE,
|
|
162
|
-
mode: 'text',
|
|
163
|
-
});
|
|
164
|
-
// Write compressed archive
|
|
165
|
-
await fs.writeFile(archivePath, compressionResult.compressed);
|
|
166
|
-
// Write metadata file
|
|
167
|
-
const metadataPath = `${archivePath}.meta.json`;
|
|
168
|
-
const metadata = {
|
|
169
|
-
timestamp: new Date().toISOString(),
|
|
170
|
-
entityCount: entities.length,
|
|
171
|
-
entityNames: entities.map(e => e.name),
|
|
172
|
-
compressed: true,
|
|
173
|
-
compressionFormat: 'brotli',
|
|
174
|
-
originalSize: compressionResult.originalSize,
|
|
175
|
-
compressedSize: compressionResult.compressedSize,
|
|
176
|
-
compressionRatio: compressionResult.ratio,
|
|
177
|
-
};
|
|
178
|
-
await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
|
|
179
|
-
return {
|
|
180
|
-
archivePath,
|
|
181
|
-
originalSize: compressionResult.originalSize,
|
|
182
|
-
compressedSize: compressionResult.compressedSize,
|
|
183
|
-
compressionRatio: compressionResult.ratio,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
/**
|
|
187
|
-
* List all available archives.
|
|
188
|
-
*
|
|
189
|
-
* @returns Array of archive information with compression details
|
|
190
|
-
*/
|
|
191
|
-
async listArchives() {
|
|
192
|
-
try {
|
|
193
|
-
try {
|
|
194
|
-
await fs.access(this.archiveDir);
|
|
195
|
-
}
|
|
196
|
-
catch {
|
|
197
|
-
return [];
|
|
198
|
-
}
|
|
199
|
-
const files = await fs.readdir(this.archiveDir);
|
|
200
|
-
const archiveFiles = files.filter(f => f.startsWith('archive_') &&
|
|
201
|
-
(f.endsWith('.jsonl') || f.endsWith('.jsonl.br')) &&
|
|
202
|
-
!f.endsWith('.meta.json'));
|
|
203
|
-
const archives = [];
|
|
204
|
-
for (const fileName of archiveFiles) {
|
|
205
|
-
const filePath = join(this.archiveDir, fileName);
|
|
206
|
-
const metadataPath = `${filePath}.meta.json`;
|
|
207
|
-
try {
|
|
208
|
-
const metadataContent = await fs.readFile(metadataPath, 'utf-8');
|
|
209
|
-
const metadata = JSON.parse(metadataContent);
|
|
210
|
-
archives.push({
|
|
211
|
-
fileName,
|
|
212
|
-
filePath,
|
|
213
|
-
timestamp: metadata.timestamp,
|
|
214
|
-
entityCount: metadata.entityCount,
|
|
215
|
-
compressed: metadata.compressed ?? fileName.endsWith('.br'),
|
|
216
|
-
originalSize: metadata.originalSize,
|
|
217
|
-
compressedSize: metadata.compressedSize,
|
|
218
|
-
compressionRatio: metadata.compressionRatio,
|
|
219
|
-
});
|
|
220
|
-
}
|
|
221
|
-
catch {
|
|
222
|
-
// Skip archives without valid metadata
|
|
223
|
-
continue;
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
// Sort by timestamp (newest first)
|
|
227
|
-
archives.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
228
|
-
return archives;
|
|
229
|
-
}
|
|
230
|
-
catch {
|
|
231
|
-
return [];
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Get the path to the archives directory.
|
|
236
|
-
*/
|
|
237
|
-
getArchiveDir() {
|
|
238
|
-
return this.archiveDir;
|
|
239
|
-
}
|
|
240
|
-
}
|