@claude-flow/memory 3.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agentic-flow/intelligence.json +16 -0
- package/README.md +249 -0
- package/__tests__/coverage/base.css +224 -0
- package/__tests__/coverage/block-navigation.js +87 -0
- package/__tests__/coverage/coverage-final.json +19 -0
- package/__tests__/coverage/favicon.png +0 -0
- package/__tests__/coverage/index.html +206 -0
- package/__tests__/coverage/lcov-report/base.css +224 -0
- package/__tests__/coverage/lcov-report/block-navigation.js +87 -0
- package/__tests__/coverage/lcov-report/favicon.png +0 -0
- package/__tests__/coverage/lcov-report/index.html +206 -0
- package/__tests__/coverage/lcov-report/prettify.css +1 -0
- package/__tests__/coverage/lcov-report/prettify.js +2 -0
- package/__tests__/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/__tests__/coverage/lcov-report/sorter.js +210 -0
- package/__tests__/coverage/lcov-report/src/agentdb-adapter.ts.html +2737 -0
- package/__tests__/coverage/lcov-report/src/agentdb-backend.ts.html +3130 -0
- package/__tests__/coverage/lcov-report/src/application/commands/delete-memory.command.ts.html +601 -0
- package/__tests__/coverage/lcov-report/src/application/commands/index.html +131 -0
- package/__tests__/coverage/lcov-report/src/application/commands/store-memory.command.ts.html +394 -0
- package/__tests__/coverage/lcov-report/src/application/queries/index.html +116 -0
- package/__tests__/coverage/lcov-report/src/application/queries/search-memory.query.ts.html +796 -0
- package/__tests__/coverage/lcov-report/src/application/services/index.html +116 -0
- package/__tests__/coverage/lcov-report/src/application/services/memory-application-service.ts.html +793 -0
- package/__tests__/coverage/lcov-report/src/cache-manager.ts.html +1633 -0
- package/__tests__/coverage/lcov-report/src/database-provider.ts.html +1618 -0
- package/__tests__/coverage/lcov-report/src/domain/entities/index.html +116 -0
- package/__tests__/coverage/lcov-report/src/domain/entities/memory-entry.ts.html +952 -0
- package/__tests__/coverage/lcov-report/src/domain/services/index.html +116 -0
- package/__tests__/coverage/lcov-report/src/domain/services/memory-domain-service.ts.html +1294 -0
- package/__tests__/coverage/lcov-report/src/hnsw-index.ts.html +3124 -0
- package/__tests__/coverage/lcov-report/src/hybrid-backend.ts.html +2167 -0
- package/__tests__/coverage/lcov-report/src/index.html +266 -0
- package/__tests__/coverage/lcov-report/src/infrastructure/repositories/hybrid-memory-repository.ts.html +1633 -0
- package/__tests__/coverage/lcov-report/src/infrastructure/repositories/index.html +116 -0
- package/__tests__/coverage/lcov-report/src/migration.ts.html +2092 -0
- package/__tests__/coverage/lcov-report/src/query-builder.ts.html +1711 -0
- package/__tests__/coverage/lcov-report/src/sqlite-backend.ts.html +2281 -0
- package/__tests__/coverage/lcov-report/src/sqljs-backend.ts.html +2374 -0
- package/__tests__/coverage/lcov-report/src/types.ts.html +2266 -0
- package/__tests__/coverage/lcov.info +10238 -0
- package/__tests__/coverage/prettify.css +1 -0
- package/__tests__/coverage/prettify.js +2 -0
- package/__tests__/coverage/sort-arrow-sprite.png +0 -0
- package/__tests__/coverage/sorter.js +210 -0
- package/__tests__/coverage/src/agentdb-adapter.ts.html +2737 -0
- package/__tests__/coverage/src/agentdb-backend.ts.html +3130 -0
- package/__tests__/coverage/src/application/commands/delete-memory.command.ts.html +601 -0
- package/__tests__/coverage/src/application/commands/index.html +131 -0
- package/__tests__/coverage/src/application/commands/store-memory.command.ts.html +394 -0
- package/__tests__/coverage/src/application/queries/index.html +116 -0
- package/__tests__/coverage/src/application/queries/search-memory.query.ts.html +796 -0
- package/__tests__/coverage/src/application/services/index.html +116 -0
- package/__tests__/coverage/src/application/services/memory-application-service.ts.html +793 -0
- package/__tests__/coverage/src/cache-manager.ts.html +1633 -0
- package/__tests__/coverage/src/database-provider.ts.html +1618 -0
- package/__tests__/coverage/src/domain/entities/index.html +116 -0
- package/__tests__/coverage/src/domain/entities/memory-entry.ts.html +952 -0
- package/__tests__/coverage/src/domain/services/index.html +116 -0
- package/__tests__/coverage/src/domain/services/memory-domain-service.ts.html +1294 -0
- package/__tests__/coverage/src/hnsw-index.ts.html +3124 -0
- package/__tests__/coverage/src/hybrid-backend.ts.html +2167 -0
- package/__tests__/coverage/src/index.html +266 -0
- package/__tests__/coverage/src/infrastructure/repositories/hybrid-memory-repository.ts.html +1633 -0
- package/__tests__/coverage/src/infrastructure/repositories/index.html +116 -0
- package/__tests__/coverage/src/migration.ts.html +2092 -0
- package/__tests__/coverage/src/query-builder.ts.html +1711 -0
- package/__tests__/coverage/src/sqlite-backend.ts.html +2281 -0
- package/__tests__/coverage/src/sqljs-backend.ts.html +2374 -0
- package/__tests__/coverage/src/types.ts.html +2266 -0
- package/benchmarks/cache-hit-rate.bench.ts +535 -0
- package/benchmarks/hnsw-indexing.bench.ts +552 -0
- package/benchmarks/memory-write.bench.ts +469 -0
- package/benchmarks/vector-search.bench.ts +449 -0
- package/dist/agentdb-adapter.d.ts +146 -0
- package/dist/agentdb-adapter.d.ts.map +1 -0
- package/dist/agentdb-adapter.js +679 -0
- package/dist/agentdb-adapter.js.map +1 -0
- package/dist/agentdb-backend.d.ts +214 -0
- package/dist/agentdb-backend.d.ts.map +1 -0
- package/dist/agentdb-backend.js +827 -0
- package/dist/agentdb-backend.js.map +1 -0
- package/dist/agentdb-backend.test.d.ts +7 -0
- package/dist/agentdb-backend.test.d.ts.map +1 -0
- package/dist/agentdb-backend.test.js +258 -0
- package/dist/agentdb-backend.test.js.map +1 -0
- package/dist/application/commands/delete-memory.command.d.ts +65 -0
- package/dist/application/commands/delete-memory.command.d.ts.map +1 -0
- package/dist/application/commands/delete-memory.command.js +129 -0
- package/dist/application/commands/delete-memory.command.js.map +1 -0
- package/dist/application/commands/store-memory.command.d.ts +48 -0
- package/dist/application/commands/store-memory.command.d.ts.map +1 -0
- package/dist/application/commands/store-memory.command.js +72 -0
- package/dist/application/commands/store-memory.command.js.map +1 -0
- package/dist/application/index.d.ts +12 -0
- package/dist/application/index.d.ts.map +1 -0
- package/dist/application/index.js +15 -0
- package/dist/application/index.js.map +1 -0
- package/dist/application/queries/search-memory.query.d.ts +72 -0
- package/dist/application/queries/search-memory.query.d.ts.map +1 -0
- package/dist/application/queries/search-memory.query.js +143 -0
- package/dist/application/queries/search-memory.query.js.map +1 -0
- package/dist/application/services/memory-application-service.d.ts +121 -0
- package/dist/application/services/memory-application-service.d.ts.map +1 -0
- package/dist/application/services/memory-application-service.js +190 -0
- package/dist/application/services/memory-application-service.js.map +1 -0
- package/dist/cache-manager.d.ts +134 -0
- package/dist/cache-manager.d.ts.map +1 -0
- package/dist/cache-manager.js +407 -0
- package/dist/cache-manager.js.map +1 -0
- package/dist/database-provider.d.ts +86 -0
- package/dist/database-provider.d.ts.map +1 -0
- package/dist/database-provider.js +385 -0
- package/dist/database-provider.js.map +1 -0
- package/dist/database-provider.test.d.ts +7 -0
- package/dist/database-provider.test.d.ts.map +1 -0
- package/dist/database-provider.test.js +285 -0
- package/dist/database-provider.test.js.map +1 -0
- package/dist/domain/entities/memory-entry.d.ts +143 -0
- package/dist/domain/entities/memory-entry.d.ts.map +1 -0
- package/dist/domain/entities/memory-entry.js +226 -0
- package/dist/domain/entities/memory-entry.js.map +1 -0
- package/dist/domain/index.d.ts +11 -0
- package/dist/domain/index.d.ts.map +1 -0
- package/dist/domain/index.js +12 -0
- package/dist/domain/index.js.map +1 -0
- package/dist/domain/repositories/memory-repository.interface.d.ts +102 -0
- package/dist/domain/repositories/memory-repository.interface.d.ts.map +1 -0
- package/dist/domain/repositories/memory-repository.interface.js +11 -0
- package/dist/domain/repositories/memory-repository.interface.js.map +1 -0
- package/dist/domain/services/memory-domain-service.d.ts +105 -0
- package/dist/domain/services/memory-domain-service.d.ts.map +1 -0
- package/dist/domain/services/memory-domain-service.js +297 -0
- package/dist/domain/services/memory-domain-service.js.map +1 -0
- package/dist/hnsw-index.d.ts +111 -0
- package/dist/hnsw-index.d.ts.map +1 -0
- package/dist/hnsw-index.js +781 -0
- package/dist/hnsw-index.js.map +1 -0
- package/dist/hybrid-backend.d.ts +217 -0
- package/dist/hybrid-backend.d.ts.map +1 -0
- package/dist/hybrid-backend.js +491 -0
- package/dist/hybrid-backend.js.map +1 -0
- package/dist/hybrid-backend.test.d.ts +8 -0
- package/dist/hybrid-backend.test.d.ts.map +1 -0
- package/dist/hybrid-backend.test.js +320 -0
- package/dist/hybrid-backend.test.js.map +1 -0
- package/dist/index.d.ts +188 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +345 -0
- package/dist/index.js.map +1 -0
- package/dist/infrastructure/index.d.ts +17 -0
- package/dist/infrastructure/index.d.ts.map +1 -0
- package/dist/infrastructure/index.js +16 -0
- package/dist/infrastructure/index.js.map +1 -0
- package/dist/infrastructure/repositories/hybrid-memory-repository.d.ts +66 -0
- package/dist/infrastructure/repositories/hybrid-memory-repository.d.ts.map +1 -0
- package/dist/infrastructure/repositories/hybrid-memory-repository.js +409 -0
- package/dist/infrastructure/repositories/hybrid-memory-repository.js.map +1 -0
- package/dist/migration.d.ts +68 -0
- package/dist/migration.d.ts.map +1 -0
- package/dist/migration.js +513 -0
- package/dist/migration.js.map +1 -0
- package/dist/query-builder.d.ts +211 -0
- package/dist/query-builder.d.ts.map +1 -0
- package/dist/query-builder.js +438 -0
- package/dist/query-builder.js.map +1 -0
- package/dist/sqlite-backend.d.ts +121 -0
- package/dist/sqlite-backend.d.ts.map +1 -0
- package/dist/sqlite-backend.js +564 -0
- package/dist/sqlite-backend.js.map +1 -0
- package/dist/sqljs-backend.d.ts +128 -0
- package/dist/sqljs-backend.d.ts.map +1 -0
- package/dist/sqljs-backend.js +598 -0
- package/dist/sqljs-backend.js.map +1 -0
- package/dist/types.d.ts +481 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +58 -0
- package/dist/types.js.map +1 -0
- package/docs/AGENTDB-INTEGRATION.md +388 -0
- package/docs/CROSS_PLATFORM.md +505 -0
- package/docs/WINDOWS_SUPPORT.md +422 -0
- package/examples/agentdb-example.ts +345 -0
- package/examples/cross-platform-usage.ts +326 -0
- package/framework/benchmark.ts +112 -0
- package/package.json +31 -0
- package/src/agentdb-adapter.ts +884 -0
- package/src/agentdb-backend.test.ts +339 -0
- package/src/agentdb-backend.ts +1016 -0
- package/src/application/commands/delete-memory.command.ts +172 -0
- package/src/application/commands/store-memory.command.ts +103 -0
- package/src/application/index.ts +36 -0
- package/src/application/queries/search-memory.query.ts +237 -0
- package/src/application/services/memory-application-service.ts +236 -0
- package/src/cache-manager.ts +516 -0
- package/src/database-provider.test.ts +364 -0
- package/src/database-provider.ts +511 -0
- package/src/domain/entities/memory-entry.ts +289 -0
- package/src/domain/index.ts +35 -0
- package/src/domain/repositories/memory-repository.interface.ts +120 -0
- package/src/domain/services/memory-domain-service.ts +403 -0
- package/src/hnsw-index.ts +1013 -0
- package/src/hybrid-backend.test.ts +399 -0
- package/src/hybrid-backend.ts +694 -0
- package/src/index.ts +515 -0
- package/src/infrastructure/index.ts +23 -0
- package/src/infrastructure/repositories/hybrid-memory-repository.ts +516 -0
- package/src/migration.ts +669 -0
- package/src/query-builder.ts +542 -0
- package/src/sqlite-backend.ts +732 -0
- package/src/sqljs-backend.ts +763 -0
- package/src/types.ts +727 -0
- package/tsconfig.json +9 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/verify-cross-platform.ts +170 -0
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Domain Service - Domain Layer
|
|
3
|
+
*
|
|
4
|
+
* Contains domain logic that doesn't naturally fit within a single entity.
|
|
5
|
+
* Coordinates between multiple memory entries and enforces domain rules.
|
|
6
|
+
*
|
|
7
|
+
* @module v3/memory/domain/services
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { MemoryEntry, MemoryType, MemoryEntryProps } from '../entities/memory-entry.js';
|
|
11
|
+
import { IMemoryRepository, VectorSearchResult } from '../repositories/memory-repository.interface.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Memory consolidation strategy
|
|
15
|
+
*/
|
|
16
|
+
export type ConsolidationStrategy = 'merge' | 'dedupe' | 'prune' | 'summarize';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Memory consolidation options
|
|
20
|
+
*/
|
|
21
|
+
export interface ConsolidationOptions {
|
|
22
|
+
strategy: ConsolidationStrategy;
|
|
23
|
+
namespace?: string;
|
|
24
|
+
threshold?: number;
|
|
25
|
+
maxAge?: number; // Maximum age in milliseconds
|
|
26
|
+
keepHot?: boolean; // Keep frequently accessed memories
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Consolidation result
|
|
31
|
+
*/
|
|
32
|
+
export interface ConsolidationResult {
|
|
33
|
+
processed: number;
|
|
34
|
+
consolidated: number;
|
|
35
|
+
removed: number;
|
|
36
|
+
newEntries: MemoryEntry[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Memory deduplication result
|
|
41
|
+
*/
|
|
42
|
+
export interface DeduplicationResult {
|
|
43
|
+
duplicatesFound: number;
|
|
44
|
+
duplicatesRemoved: number;
|
|
45
|
+
groupsProcessed: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Memory namespace statistics
|
|
50
|
+
*/
|
|
51
|
+
export interface NamespaceAnalysis {
|
|
52
|
+
namespace: string;
|
|
53
|
+
totalEntries: number;
|
|
54
|
+
activeEntries: number;
|
|
55
|
+
totalSize: number;
|
|
56
|
+
averageAccessCount: number;
|
|
57
|
+
oldestEntry: Date;
|
|
58
|
+
newestEntry: Date;
|
|
59
|
+
typeDistribution: Record<MemoryType, number>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Memory Domain Service
|
|
64
|
+
*
|
|
65
|
+
* Provides domain-level operations that span multiple entities.
|
|
66
|
+
* Implements business rules for memory management.
|
|
67
|
+
*/
|
|
68
|
+
export class MemoryDomainService {
|
|
69
|
+
constructor(private readonly repository: IMemoryRepository) {}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Store a new memory with automatic type detection
|
|
73
|
+
*/
|
|
74
|
+
async storeWithTypeDetection(
|
|
75
|
+
namespace: string,
|
|
76
|
+
key: string,
|
|
77
|
+
value: unknown,
|
|
78
|
+
vector?: Float32Array
|
|
79
|
+
): Promise<MemoryEntry> {
|
|
80
|
+
const type = this.detectMemoryType(value);
|
|
81
|
+
const entry = MemoryEntry.create({
|
|
82
|
+
namespace,
|
|
83
|
+
key,
|
|
84
|
+
value,
|
|
85
|
+
type,
|
|
86
|
+
vector,
|
|
87
|
+
});
|
|
88
|
+
await this.repository.save(entry);
|
|
89
|
+
return entry;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Retrieve and record access
|
|
94
|
+
*/
|
|
95
|
+
async retrieveWithAccessTracking(
|
|
96
|
+
namespace: string,
|
|
97
|
+
key: string
|
|
98
|
+
): Promise<MemoryEntry | null> {
|
|
99
|
+
const entry = await this.repository.findByKey(namespace, key);
|
|
100
|
+
if (entry && entry.isAccessible()) {
|
|
101
|
+
entry.recordAccess();
|
|
102
|
+
await this.repository.save(entry);
|
|
103
|
+
}
|
|
104
|
+
return entry;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Search for similar memories and record access
|
|
109
|
+
*/
|
|
110
|
+
async searchSimilarWithTracking(
|
|
111
|
+
vector: Float32Array,
|
|
112
|
+
namespace?: string,
|
|
113
|
+
limit: number = 10
|
|
114
|
+
): Promise<VectorSearchResult[]> {
|
|
115
|
+
const results = await this.repository.searchByVector({
|
|
116
|
+
vector,
|
|
117
|
+
namespace,
|
|
118
|
+
limit,
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// Record access for all returned entries
|
|
122
|
+
await Promise.all(
|
|
123
|
+
results.map(async (result) => {
|
|
124
|
+
result.entry.recordAccess();
|
|
125
|
+
await this.repository.save(result.entry);
|
|
126
|
+
})
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
return results;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Consolidate memories based on strategy
|
|
134
|
+
*/
|
|
135
|
+
async consolidate(options: ConsolidationOptions): Promise<ConsolidationResult> {
|
|
136
|
+
const entries = await this.repository.findByNamespace(
|
|
137
|
+
options.namespace ?? 'default',
|
|
138
|
+
{ status: 'active' }
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
let result: ConsolidationResult = {
|
|
142
|
+
processed: entries.length,
|
|
143
|
+
consolidated: 0,
|
|
144
|
+
removed: 0,
|
|
145
|
+
newEntries: [],
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
switch (options.strategy) {
|
|
149
|
+
case 'prune':
|
|
150
|
+
result = await this.pruneOldMemories(entries, options);
|
|
151
|
+
break;
|
|
152
|
+
case 'dedupe':
|
|
153
|
+
const dedupeResult = await this.deduplicateMemories(entries, options);
|
|
154
|
+
result.removed = dedupeResult.duplicatesRemoved;
|
|
155
|
+
break;
|
|
156
|
+
case 'merge':
|
|
157
|
+
result = await this.mergeRelatedMemories(entries, options);
|
|
158
|
+
break;
|
|
159
|
+
default:
|
|
160
|
+
// No-op for unknown strategies
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return result;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Detect memory type based on value structure
|
|
168
|
+
*/
|
|
169
|
+
private detectMemoryType(value: unknown): MemoryType {
|
|
170
|
+
if (typeof value === 'string') {
|
|
171
|
+
// Long text is likely semantic
|
|
172
|
+
if (value.length > 500) return 'semantic';
|
|
173
|
+
// Short instructions are procedural
|
|
174
|
+
if (value.includes('->') || value.includes('then')) return 'procedural';
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
if (typeof value === 'object' && value !== null) {
|
|
178
|
+
const obj = value as Record<string, unknown>;
|
|
179
|
+
// Objects with timestamps are episodic
|
|
180
|
+
if ('timestamp' in obj || 'when' in obj || 'date' in obj) return 'episodic';
|
|
181
|
+
// Objects with steps are procedural
|
|
182
|
+
if ('steps' in obj || 'actions' in obj) return 'procedural';
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Default to working memory for short-term storage
|
|
186
|
+
return 'working';
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Prune old, rarely accessed memories
|
|
191
|
+
*/
|
|
192
|
+
private async pruneOldMemories(
|
|
193
|
+
entries: MemoryEntry[],
|
|
194
|
+
options: ConsolidationOptions
|
|
195
|
+
): Promise<ConsolidationResult> {
|
|
196
|
+
const maxAge = options.maxAge ?? 7 * 24 * 60 * 60 * 1000; // 7 days default
|
|
197
|
+
const threshold = options.threshold ?? 5; // Minimum access count to keep
|
|
198
|
+
const now = Date.now();
|
|
199
|
+
const toRemove: string[] = [];
|
|
200
|
+
|
|
201
|
+
for (const entry of entries) {
|
|
202
|
+
const age = now - entry.createdAt.getTime();
|
|
203
|
+
const isOld = age > maxAge;
|
|
204
|
+
const isRarelyAccessed = entry.accessCount < threshold;
|
|
205
|
+
|
|
206
|
+
// Keep hot memories if requested
|
|
207
|
+
if (options.keepHot && entry.isHot()) continue;
|
|
208
|
+
|
|
209
|
+
if (isOld && isRarelyAccessed) {
|
|
210
|
+
entry.archive();
|
|
211
|
+
toRemove.push(entry.id);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (toRemove.length > 0) {
|
|
216
|
+
await this.repository.deleteMany(toRemove);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
return {
|
|
220
|
+
processed: entries.length,
|
|
221
|
+
consolidated: 0,
|
|
222
|
+
removed: toRemove.length,
|
|
223
|
+
newEntries: [],
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Find and remove duplicate memories
|
|
229
|
+
*/
|
|
230
|
+
private async deduplicateMemories(
|
|
231
|
+
entries: MemoryEntry[],
|
|
232
|
+
options: ConsolidationOptions
|
|
233
|
+
): Promise<DeduplicationResult> {
|
|
234
|
+
const threshold = options.threshold ?? 0.95; // Similarity threshold
|
|
235
|
+
const duplicates: string[] = [];
|
|
236
|
+
const processed = new Set<string>();
|
|
237
|
+
|
|
238
|
+
for (let i = 0; i < entries.length; i++) {
|
|
239
|
+
if (processed.has(entries[i].id)) continue;
|
|
240
|
+
|
|
241
|
+
const entry = entries[i];
|
|
242
|
+
if (!entry.vector) continue;
|
|
243
|
+
|
|
244
|
+
// Find similar entries
|
|
245
|
+
const similar = await this.repository.searchByVector({
|
|
246
|
+
vector: entry.vector,
|
|
247
|
+
namespace: entry.namespace,
|
|
248
|
+
limit: 10,
|
|
249
|
+
threshold,
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
// Mark duplicates (keep the one with highest access count)
|
|
253
|
+
const group = similar
|
|
254
|
+
.filter((s) => s.entry.id !== entry.id && s.similarity >= threshold)
|
|
255
|
+
.sort((a, b) => b.entry.accessCount - a.entry.accessCount);
|
|
256
|
+
|
|
257
|
+
for (const dup of group.slice(1)) {
|
|
258
|
+
duplicates.push(dup.entry.id);
|
|
259
|
+
processed.add(dup.entry.id);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
processed.add(entry.id);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (duplicates.length > 0) {
|
|
266
|
+
await this.repository.deleteMany(duplicates);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return {
|
|
270
|
+
duplicatesFound: duplicates.length,
|
|
271
|
+
duplicatesRemoved: duplicates.length,
|
|
272
|
+
groupsProcessed: processed.size,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Merge related memories into consolidated entries
|
|
278
|
+
*/
|
|
279
|
+
private async mergeRelatedMemories(
|
|
280
|
+
entries: MemoryEntry[],
|
|
281
|
+
options: ConsolidationOptions
|
|
282
|
+
): Promise<ConsolidationResult> {
|
|
283
|
+
const threshold = options.threshold ?? 0.8;
|
|
284
|
+
const newEntries: MemoryEntry[] = [];
|
|
285
|
+
const toRemove: string[] = [];
|
|
286
|
+
const processed = new Set<string>();
|
|
287
|
+
|
|
288
|
+
for (const entry of entries) {
|
|
289
|
+
if (processed.has(entry.id)) continue;
|
|
290
|
+
if (!entry.vector) {
|
|
291
|
+
processed.add(entry.id);
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// Find related entries
|
|
296
|
+
const related = await this.repository.searchByVector({
|
|
297
|
+
vector: entry.vector,
|
|
298
|
+
namespace: entry.namespace,
|
|
299
|
+
limit: 5,
|
|
300
|
+
threshold,
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
if (related.length > 1) {
|
|
304
|
+
// Merge related entries
|
|
305
|
+
const merged = this.mergeEntries(related.map((r) => r.entry));
|
|
306
|
+
newEntries.push(merged);
|
|
307
|
+
|
|
308
|
+
for (const r of related) {
|
|
309
|
+
toRemove.push(r.entry.id);
|
|
310
|
+
processed.add(r.entry.id);
|
|
311
|
+
}
|
|
312
|
+
} else {
|
|
313
|
+
processed.add(entry.id);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Remove old entries and save merged ones
|
|
318
|
+
if (toRemove.length > 0) {
|
|
319
|
+
await this.repository.deleteMany(toRemove);
|
|
320
|
+
}
|
|
321
|
+
if (newEntries.length > 0) {
|
|
322
|
+
await this.repository.saveMany(newEntries);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
return {
|
|
326
|
+
processed: entries.length,
|
|
327
|
+
consolidated: newEntries.length,
|
|
328
|
+
removed: toRemove.length,
|
|
329
|
+
newEntries,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Merge multiple entries into one
|
|
335
|
+
*/
|
|
336
|
+
private mergeEntries(entries: MemoryEntry[]): MemoryEntry {
|
|
337
|
+
// Sort by access count to prioritize most accessed
|
|
338
|
+
const sorted = [...entries].sort((a, b) => b.accessCount - a.accessCount);
|
|
339
|
+
const primary = sorted[0];
|
|
340
|
+
|
|
341
|
+
// Combine metadata
|
|
342
|
+
const combinedMetadata: Record<string, unknown> = {};
|
|
343
|
+
for (const entry of entries) {
|
|
344
|
+
Object.assign(combinedMetadata, entry.metadata);
|
|
345
|
+
}
|
|
346
|
+
combinedMetadata.mergedFrom = entries.map((e) => e.id);
|
|
347
|
+
combinedMetadata.mergedAt = new Date().toISOString();
|
|
348
|
+
|
|
349
|
+
// Create merged entry
|
|
350
|
+
return MemoryEntry.create({
|
|
351
|
+
namespace: primary.namespace,
|
|
352
|
+
key: `merged_${Date.now()}`,
|
|
353
|
+
value: {
|
|
354
|
+
primary: primary.value,
|
|
355
|
+
related: sorted.slice(1).map((e) => e.value),
|
|
356
|
+
},
|
|
357
|
+
type: primary.type,
|
|
358
|
+
vector: primary.vector,
|
|
359
|
+
metadata: combinedMetadata,
|
|
360
|
+
accessCount: entries.reduce((sum, e) => sum + e.accessCount, 0),
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Analyze a namespace
|
|
366
|
+
*/
|
|
367
|
+
async analyzeNamespace(namespace: string): Promise<NamespaceAnalysis> {
|
|
368
|
+
const entries = await this.repository.findByNamespace(namespace);
|
|
369
|
+
const active = entries.filter((e) => e.status === 'active');
|
|
370
|
+
|
|
371
|
+
const typeDistribution: Record<MemoryType, number> = {
|
|
372
|
+
semantic: 0,
|
|
373
|
+
episodic: 0,
|
|
374
|
+
procedural: 0,
|
|
375
|
+
working: 0,
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
let totalAccessCount = 0;
|
|
379
|
+
let totalSize = 0;
|
|
380
|
+
let oldestDate = new Date();
|
|
381
|
+
let newestDate = new Date(0);
|
|
382
|
+
|
|
383
|
+
for (const entry of entries) {
|
|
384
|
+
typeDistribution[entry.type]++;
|
|
385
|
+
totalAccessCount += entry.accessCount;
|
|
386
|
+
totalSize += JSON.stringify(entry.value).length;
|
|
387
|
+
|
|
388
|
+
if (entry.createdAt < oldestDate) oldestDate = entry.createdAt;
|
|
389
|
+
if (entry.createdAt > newestDate) newestDate = entry.createdAt;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
return {
|
|
393
|
+
namespace,
|
|
394
|
+
totalEntries: entries.length,
|
|
395
|
+
activeEntries: active.length,
|
|
396
|
+
totalSize,
|
|
397
|
+
averageAccessCount: entries.length > 0 ? totalAccessCount / entries.length : 0,
|
|
398
|
+
oldestEntry: oldestDate,
|
|
399
|
+
newestEntry: newestDate,
|
|
400
|
+
typeDistribution,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
}
|