@claude-flow/memory 3.0.0-alpha.9 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +308 -37
  2. package/dist/agent-memory-scope.d.ts.map +1 -1
  3. package/dist/agent-memory-scope.js +10 -2
  4. package/dist/agent-memory-scope.js.map +1 -1
  5. package/dist/agentdb-adapter-key-isolation.test.d.ts +2 -0
  6. package/dist/agentdb-adapter-key-isolation.test.d.ts.map +1 -0
  7. package/dist/agentdb-adapter-key-isolation.test.js +175 -0
  8. package/dist/agentdb-adapter-key-isolation.test.js.map +1 -0
  9. package/dist/agentdb-adapter-upsert.test.d.ts +2 -0
  10. package/dist/agentdb-adapter-upsert.test.d.ts.map +1 -0
  11. package/dist/agentdb-adapter-upsert.test.js +216 -0
  12. package/dist/agentdb-adapter-upsert.test.js.map +1 -0
  13. package/dist/agentdb-adapter.d.ts +70 -1
  14. package/dist/agentdb-adapter.d.ts.map +1 -1
  15. package/dist/agentdb-adapter.js +365 -32
  16. package/dist/agentdb-adapter.js.map +1 -1
  17. package/dist/agentdb-backend.d.ts +25 -0
  18. package/dist/agentdb-backend.d.ts.map +1 -1
  19. package/dist/agentdb-backend.js +113 -28
  20. package/dist/agentdb-backend.js.map +1 -1
  21. package/dist/agentdb-retrieval-guard.d.ts +76 -0
  22. package/dist/agentdb-retrieval-guard.d.ts.map +1 -0
  23. package/dist/agentdb-retrieval-guard.js +77 -0
  24. package/dist/agentdb-retrieval-guard.js.map +1 -0
  25. package/dist/agentdb-retrieval-guard.test.d.ts +12 -0
  26. package/dist/agentdb-retrieval-guard.test.d.ts.map +1 -0
  27. package/dist/agentdb-retrieval-guard.test.js +129 -0
  28. package/dist/agentdb-retrieval-guard.test.js.map +1 -0
  29. package/dist/auto-memory-bridge.d.ts +12 -1
  30. package/dist/auto-memory-bridge.d.ts.map +1 -1
  31. package/dist/auto-memory-bridge.js +101 -7
  32. package/dist/auto-memory-bridge.js.map +1 -1
  33. package/dist/auto-memory-bridge.test.js +121 -15
  34. package/dist/auto-memory-bridge.test.js.map +1 -1
  35. package/dist/auto-memory-index-3224.test.d.ts +2 -0
  36. package/dist/auto-memory-index-3224.test.d.ts.map +1 -0
  37. package/dist/auto-memory-index-3224.test.js +77 -0
  38. package/dist/auto-memory-index-3224.test.js.map +1 -0
  39. package/dist/consolidator-embedding-benchmark.test.d.ts +39 -0
  40. package/dist/consolidator-embedding-benchmark.test.d.ts.map +1 -0
  41. package/dist/consolidator-embedding-benchmark.test.js +366 -0
  42. package/dist/consolidator-embedding-benchmark.test.js.map +1 -0
  43. package/dist/consolidator.d.ts +131 -0
  44. package/dist/consolidator.d.ts.map +1 -0
  45. package/dist/consolidator.js +309 -0
  46. package/dist/consolidator.js.map +1 -0
  47. package/dist/consolidator.test.d.ts +17 -0
  48. package/dist/consolidator.test.d.ts.map +1 -0
  49. package/dist/consolidator.test.js +376 -0
  50. package/dist/consolidator.test.js.map +1 -0
  51. package/dist/controller-registry.d.ts +290 -0
  52. package/dist/controller-registry.d.ts.map +1 -0
  53. package/dist/controller-registry.js +1095 -0
  54. package/dist/controller-registry.js.map +1 -0
  55. package/dist/controller-registry.test.d.ts +14 -0
  56. package/dist/controller-registry.test.d.ts.map +1 -0
  57. package/dist/controller-registry.test.js +818 -0
  58. package/dist/controller-registry.test.js.map +1 -0
  59. package/dist/database-provider.d.ts +14 -2
  60. package/dist/database-provider.d.ts.map +1 -1
  61. package/dist/database-provider.js +63 -3
  62. package/dist/database-provider.js.map +1 -1
  63. package/dist/database-provider.test.js +56 -3
  64. package/dist/database-provider.test.js.map +1 -1
  65. package/dist/entity-tagger.d.ts +31 -0
  66. package/dist/entity-tagger.d.ts.map +1 -0
  67. package/dist/entity-tagger.js +80 -0
  68. package/dist/entity-tagger.js.map +1 -0
  69. package/dist/entity-tagger.test.d.ts +12 -0
  70. package/dist/entity-tagger.test.d.ts.map +1 -0
  71. package/dist/entity-tagger.test.js +70 -0
  72. package/dist/entity-tagger.test.js.map +1 -0
  73. package/dist/fts5.test.d.ts +13 -0
  74. package/dist/fts5.test.d.ts.map +1 -0
  75. package/dist/fts5.test.js +144 -0
  76. package/dist/fts5.test.js.map +1 -0
  77. package/dist/graceful-retrieval.test.d.ts +14 -0
  78. package/dist/graceful-retrieval.test.d.ts.map +1 -0
  79. package/dist/graceful-retrieval.test.js +251 -0
  80. package/dist/graceful-retrieval.test.js.map +1 -0
  81. package/dist/hnsw-index.d.ts +62 -0
  82. package/dist/hnsw-index.d.ts.map +1 -1
  83. package/dist/hnsw-index.js +461 -16
  84. package/dist/hnsw-index.js.map +1 -1
  85. package/dist/hnsw-persistence.test.d.ts +13 -0
  86. package/dist/hnsw-persistence.test.d.ts.map +1 -0
  87. package/dist/hnsw-persistence.test.js +208 -0
  88. package/dist/hnsw-persistence.test.js.map +1 -0
  89. package/dist/hnsw-quantization.test.d.ts +17 -0
  90. package/dist/hnsw-quantization.test.d.ts.map +1 -0
  91. package/dist/hnsw-quantization.test.js +140 -0
  92. package/dist/hnsw-quantization.test.js.map +1 -0
  93. package/dist/hybrid-backend.d.ts +46 -2
  94. package/dist/hybrid-backend.d.ts.map +1 -1
  95. package/dist/hybrid-backend.js +120 -21
  96. package/dist/hybrid-backend.js.map +1 -1
  97. package/dist/hybrid-backend.test.js +139 -5
  98. package/dist/hybrid-backend.test.js.map +1 -1
  99. package/dist/index.d.ts +164 -10
  100. package/dist/index.d.ts.map +1 -1
  101. package/dist/index.js +272 -30
  102. package/dist/index.js.map +1 -1
  103. package/dist/index.test.d.ts +10 -0
  104. package/dist/index.test.d.ts.map +1 -0
  105. package/dist/index.test.js +71 -0
  106. package/dist/index.test.js.map +1 -0
  107. package/dist/json-security.d.ts +12 -0
  108. package/dist/json-security.d.ts.map +1 -0
  109. package/dist/json-security.js +20 -0
  110. package/dist/json-security.js.map +1 -0
  111. package/dist/learning-bridge.d.ts +29 -0
  112. package/dist/learning-bridge.d.ts.map +1 -1
  113. package/dist/learning-bridge.js +71 -6
  114. package/dist/learning-bridge.js.map +1 -1
  115. package/dist/learning-bridge.test.js +181 -1
  116. package/dist/learning-bridge.test.js.map +1 -1
  117. package/dist/memory-key.d.ts +3 -0
  118. package/dist/memory-key.d.ts.map +1 -0
  119. package/dist/memory-key.js +5 -0
  120. package/dist/memory-key.js.map +1 -0
  121. package/dist/migration.d.ts +19 -4
  122. package/dist/migration.d.ts.map +1 -1
  123. package/dist/migration.js +99 -24
  124. package/dist/migration.js.map +1 -1
  125. package/dist/migration.test.d.ts +11 -0
  126. package/dist/migration.test.d.ts.map +1 -0
  127. package/dist/migration.test.js +141 -0
  128. package/dist/migration.test.js.map +1 -0
  129. package/dist/mmr-benchmark.test.d.ts +2 -0
  130. package/dist/mmr-benchmark.test.d.ts.map +1 -0
  131. package/dist/mmr-benchmark.test.js +286 -0
  132. package/dist/mmr-benchmark.test.js.map +1 -0
  133. package/dist/mmr-rerank-perf.test.d.ts +2 -0
  134. package/dist/mmr-rerank-perf.test.d.ts.map +1 -0
  135. package/dist/mmr-rerank-perf.test.js +191 -0
  136. package/dist/mmr-rerank-perf.test.js.map +1 -0
  137. package/dist/persistent-sona.d.ts +144 -0
  138. package/dist/persistent-sona.d.ts.map +1 -0
  139. package/dist/persistent-sona.js +332 -0
  140. package/dist/persistent-sona.js.map +1 -0
  141. package/dist/rvf-backend.d.ts +53 -0
  142. package/dist/rvf-backend.d.ts.map +1 -0
  143. package/dist/rvf-backend.js +643 -0
  144. package/dist/rvf-backend.js.map +1 -0
  145. package/dist/rvf-learning-store.d.ts +139 -0
  146. package/dist/rvf-learning-store.d.ts.map +1 -0
  147. package/dist/rvf-learning-store.js +295 -0
  148. package/dist/rvf-learning-store.js.map +1 -0
  149. package/dist/rvf-migration.d.ts +45 -0
  150. package/dist/rvf-migration.d.ts.map +1 -0
  151. package/dist/rvf-migration.js +254 -0
  152. package/dist/rvf-migration.js.map +1 -0
  153. package/dist/smart-retrieval-scores.test.d.ts +2 -0
  154. package/dist/smart-retrieval-scores.test.d.ts.map +1 -0
  155. package/dist/smart-retrieval-scores.test.js +75 -0
  156. package/dist/smart-retrieval-scores.test.js.map +1 -0
  157. package/dist/smart-retrieval.d.ts +141 -0
  158. package/dist/smart-retrieval.d.ts.map +1 -0
  159. package/dist/smart-retrieval.js +389 -0
  160. package/dist/smart-retrieval.js.map +1 -0
  161. package/dist/smart-retrieval.test.d.ts +2 -0
  162. package/dist/smart-retrieval.test.d.ts.map +1 -0
  163. package/dist/smart-retrieval.test.js +418 -0
  164. package/dist/smart-retrieval.test.js.map +1 -0
  165. package/dist/sqlite-backend.d.ts +10 -0
  166. package/dist/sqlite-backend.d.ts.map +1 -1
  167. package/dist/sqlite-backend.js +101 -5
  168. package/dist/sqlite-backend.js.map +1 -1
  169. package/dist/sqljs-backend.d.ts +9 -0
  170. package/dist/sqljs-backend.d.ts.map +1 -1
  171. package/dist/sqljs-backend.js +91 -3
  172. package/dist/sqljs-backend.js.map +1 -1
  173. package/dist/tiered-memory.d.ts +171 -0
  174. package/dist/tiered-memory.d.ts.map +1 -0
  175. package/dist/tiered-memory.js +350 -0
  176. package/dist/tiered-memory.js.map +1 -0
  177. package/dist/tiered-memory.test.d.ts +9 -0
  178. package/dist/tiered-memory.test.d.ts.map +1 -0
  179. package/dist/tiered-memory.test.js +220 -0
  180. package/dist/tiered-memory.test.js.map +1 -0
  181. package/dist/types.d.ts +17 -0
  182. package/dist/types.d.ts.map +1 -1
  183. package/dist/types.js.map +1 -1
  184. package/package.json +10 -8
@@ -0,0 +1,309 @@
1
+ /**
2
+ * MemoryConsolidator — ADR-125 Phase 4
3
+ *
4
+ * Periodic maintenance for the in-memory state owned by {@link UnifiedMemoryService}.
5
+ *
6
+ * Operations:
7
+ * - `sweepExpired()` — drop entries past `expiresAt` from all indexes (including HNSW).
8
+ * - `dedup(strategy)` — collapse content-hash duplicates per strategy.
9
+ * - `compactHnsw()` — rebuild the HNSW index from current `entries`.
10
+ * - `runAll()` — sweep → dedup → compact in order.
11
+ *
12
+ * Invoked from two paths:
13
+ * 1. {@link UnifiedMemoryService}'s lifecycle (background timer when
14
+ * `consolidator.autoRun === true`, plus `close()`).
15
+ * 2. The AgentDB `nightlyLearner` controller (`src/controller-registry.ts`)
16
+ * delegates to `runAll()` instead of hitting AgentDB directly.
17
+ *
18
+ * Phase 3 placeholder lives in this file too — it provides the typed
19
+ * surface that {@link UnifiedMemoryService.getConsolidator} resolves to.
20
+ * The real implementation lands with Phase 4.
21
+ *
22
+ * @module v3/memory/consolidator
23
+ */
24
+ import { createHash } from 'node:crypto';
25
+ import { HNSWIndex } from './hnsw-index.js';
26
+ import { encodeMemoryKey } from './memory-key.js';
27
+ const DEFAULT_SIMILARITY_THRESHOLD = 0.95;
28
+ /** Neighborhood size for the near-duplicate HNSW lookup — small and cheap. */
29
+ const NEAR_DUP_SEARCH_K = 8;
30
+ /**
31
+ * Explicit `ef` (candidate-list size) for the near-duplicate HNSW search.
32
+ * Left unset, `HNSWIndex.search()` defaults `ef` to
33
+ * `Math.max(k, efConstruction)` — efConstruction defaults to 200, so an
34
+ * unset `ef` here means every dedup() search traverses a 200-candidate
35
+ * list even though duplicate detection only needs to find very-close
36
+ * neighbors (high similarity), not broad top-k recall. Measured impact at
37
+ * N=5000 (`consolidator-embedding-benchmark.test.ts`): ~9.8s with the
38
+ * inherited ef=200 default vs a small fixed ef here. 32 is a deliberate
39
+ * margin above NEAR_DUP_SEARCH_K (8) for HNSW's approximate-search
40
+ * headroom, not a re-derivation of efConstruction's separate
41
+ * index-build-quality concern.
42
+ */
43
+ const NEAR_DUP_SEARCH_EF = 32;
44
+ export class MemoryConsolidator {
45
+ service;
46
+ opts;
47
+ constructor(service, opts = {}) {
48
+ this.service = service;
49
+ this.opts = opts;
50
+ }
51
+ /**
52
+ * Drop all entries whose `expiresAt` is in the past.
53
+ */
54
+ async sweepExpired() {
55
+ const adapter = this.service.getAdapter();
56
+ const entries = adapter.entries;
57
+ const namespaceIndex = adapter.namespaceIndex;
58
+ const keyIndex = adapter.keyIndex;
59
+ const tagIndex = adapter.tagIndex;
60
+ const index = adapter.index;
61
+ const now = Date.now();
62
+ const toRemove = [];
63
+ for (const entry of entries.values()) {
64
+ if (entry.expiresAt != null && entry.expiresAt < now) {
65
+ toRemove.push(entry);
66
+ }
67
+ }
68
+ let hnswRemoved = 0;
69
+ for (const entry of toRemove) {
70
+ entries.delete(entry.id);
71
+ namespaceIndex.get(entry.namespace)?.delete(entry.id);
72
+ const compositeKey = encodeMemoryKey(entry.namespace, entry.key, adapter.config?.defaultNamespace);
73
+ if (keyIndex.get(compositeKey) === entry.id)
74
+ keyIndex.delete(compositeKey);
75
+ for (const tag of entry.tags)
76
+ tagIndex.get(tag)?.delete(entry.id);
77
+ if (entry.embedding) {
78
+ const removed = await index.removePoint(entry.id);
79
+ if (removed)
80
+ hnswRemoved += 1;
81
+ }
82
+ }
83
+ // Clean up empty namespace/tag sets to keep memory bounded
84
+ for (const [ns, ids] of namespaceIndex) {
85
+ if (ids.size === 0)
86
+ namespaceIndex.delete(ns);
87
+ }
88
+ for (const [tag, ids] of tagIndex) {
89
+ if (ids.size === 0)
90
+ tagIndex.delete(tag);
91
+ }
92
+ return {
93
+ removed: toRemove.length,
94
+ remaining: entries.size,
95
+ hnswRemoved,
96
+ };
97
+ }
98
+ /**
99
+ * Pick the keeper for a duplicate group per `strategy` and, for
100
+ * `merge-tags`, union the group's tag sets onto it. Shared by both the
101
+ * content-hash pass and the embedding near-duplicate pass in `dedup()`.
102
+ */
103
+ selectKeeper(bucket, strategy) {
104
+ const keeper = strategy === 'keep-oldest'
105
+ ? bucket.reduce((acc, e) => (e.createdAt < acc.createdAt ? e : acc))
106
+ : // keep-newest + merge-tags share this branch
107
+ bucket.reduce((acc, e) => (e.updatedAt > acc.updatedAt ? e : acc));
108
+ return keeper;
109
+ }
110
+ /**
111
+ * Drop every entry in `bucket` except the strategy-selected keeper,
112
+ * updating all adapter indexes (and the HNSW index, for entries that have
113
+ * an embedding) to match. Shared by both dedup passes.
114
+ */
115
+ async mergeGroup(bucket, strategy, ctx) {
116
+ const keeper = this.selectKeeper(bucket, strategy);
117
+ if (strategy === 'merge-tags') {
118
+ const tagSet = new Set(keeper.tags);
119
+ for (const e of bucket)
120
+ for (const t of e.tags)
121
+ tagSet.add(t);
122
+ const oldTags = keeper.tags;
123
+ keeper.tags = [...tagSet];
124
+ // Update tagIndex membership for newly-added tags
125
+ for (const t of keeper.tags) {
126
+ if (!oldTags.includes(t)) {
127
+ if (!ctx.tagIndex.has(t))
128
+ ctx.tagIndex.set(t, new Set());
129
+ ctx.tagIndex.get(t).add(keeper.id);
130
+ }
131
+ }
132
+ }
133
+ let dropped = 0;
134
+ for (const e of bucket) {
135
+ if (e.id === keeper.id)
136
+ continue;
137
+ ctx.entries.delete(e.id);
138
+ ctx.namespaceIndex.get(e.namespace)?.delete(e.id);
139
+ const compositeKey = encodeMemoryKey(e.namespace, e.key, ctx.defaultNamespace);
140
+ if (ctx.keyIndex.get(compositeKey) === e.id) {
141
+ ctx.keyIndex.delete(compositeKey);
142
+ }
143
+ for (const tag of e.tags)
144
+ ctx.tagIndex.get(tag)?.delete(e.id);
145
+ if (e.embedding) {
146
+ await ctx.index.removePoint(e.id);
147
+ }
148
+ dropped += 1;
149
+ }
150
+ return { keeperId: keeper.id, dropped };
151
+ }
152
+ /**
153
+ * Collapse duplicates across all namespaces per `strategy`, in two passes:
154
+ *
155
+ * 1. Byte-exact content-hash buckets (unchanged from pre-4.1 behavior).
156
+ * 2. Embedding near-duplicates: for entries that survive pass 1 and carry
157
+ * an `embedding`, query the already-populated `HNSWIndex` (the same
158
+ * index instance this method already holds a handle to for removal
159
+ * bookkeeping) for cosine-similarity neighbors above
160
+ * `opts.similarityThreshold` (default 0.95). This catches paraphrases
161
+ * and reformattings that byte-exact hashing structurally cannot, at no
162
+ * extra backend round-trip cost — the embedding and the index are both
163
+ * already resident in memory at this call site.
164
+ *
165
+ * Pass 2 only runs when the index's configured metric is `'cosine'` (its
166
+ * distances are `1 - similarity`) and the threshold is `< 1`; otherwise
167
+ * behavior is identical to hash-only dedup.
168
+ *
169
+ * - `keep-newest`: keep the entry with the highest `updatedAt`, drop the rest.
170
+ * - `keep-oldest`: keep the entry with the lowest `createdAt`, drop the rest.
171
+ * - `merge-tags`: keep newest, but union the tag sets of the duplicates first.
172
+ */
173
+ async dedup(strategy) {
174
+ const adapter = this.service.getAdapter();
175
+ const entries = adapter.entries;
176
+ const namespaceIndex = adapter.namespaceIndex;
177
+ const keyIndex = adapter.keyIndex;
178
+ const tagIndex = adapter.tagIndex;
179
+ const index = adapter.index;
180
+ const ctx = { entries, namespaceIndex, keyIndex, tagIndex, index, defaultNamespace: adapter.config?.defaultNamespace };
181
+ const effective = strategy ?? this.opts.dedupStrategy ?? 'keep-newest';
182
+ // Pass 1: bucket by content hash (byte-exact duplicates)
183
+ const buckets = new Map();
184
+ for (const entry of entries.values()) {
185
+ const hash = createHash('sha256').update(entry.content).digest('hex');
186
+ const bucket = buckets.get(hash);
187
+ if (bucket)
188
+ bucket.push(entry);
189
+ else
190
+ buckets.set(hash, [entry]);
191
+ }
192
+ let merged = 0;
193
+ let dupGroups = 0;
194
+ for (const bucket of buckets.values()) {
195
+ if (bucket.length <= 1)
196
+ continue;
197
+ dupGroups += 1;
198
+ const { dropped } = await this.mergeGroup(bucket, effective, ctx);
199
+ merged += dropped;
200
+ }
201
+ // Pass 2: embedding near-duplicates among pass-1 survivors.
202
+ //
203
+ // Each round only groups an entry with what its single
204
+ // `NEAR_DUP_SEARCH_K`-wide HNSW query returns, so a duplicate cluster
205
+ // larger than that window can split into more than one surviving
206
+ // sub-group in a single round (each sub-group's keeper remains
207
+ // un-merged with the others). Looping to a fixed point — re-scanning
208
+ // until a full round produces zero merges — guarantees a single
209
+ // `dedup()` call fully converges regardless of cluster size, at the
210
+ // cost of one extra full round whenever a cluster does split (bounded:
211
+ // strictly fewer entries remain each round that merges anything, so
212
+ // this always terminates). `groups` therefore counts merge operations
213
+ // across all rounds, which can exceed the number of ultimate
214
+ // underlying duplicate clusters when one of them needed more than one
215
+ // round to fully collapse.
216
+ const threshold = this.opts.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD;
217
+ if (index.getConfig().metric === 'cosine' && threshold < 1) {
218
+ let roundMerged;
219
+ do {
220
+ roundMerged = 0;
221
+ const consumed = new Set();
222
+ // Snapshot survivors up front — mutated further as groups merge.
223
+ for (const entry of [...entries.values()]) {
224
+ if (consumed.has(entry.id) || !entry.embedding || !entries.has(entry.id)) {
225
+ continue;
226
+ }
227
+ const hits = await index.search(entry.embedding, NEAR_DUP_SEARCH_K, NEAR_DUP_SEARCH_EF);
228
+ const group = [entry];
229
+ for (const hit of hits) {
230
+ if (hit.id === entry.id || consumed.has(hit.id))
231
+ continue;
232
+ const candidate = entries.get(hit.id);
233
+ if (!candidate || !candidate.embedding)
234
+ continue;
235
+ const similarity = 1 - hit.distance;
236
+ if (similarity >= threshold)
237
+ group.push(candidate);
238
+ }
239
+ if (group.length <= 1) {
240
+ consumed.add(entry.id);
241
+ continue;
242
+ }
243
+ dupGroups += 1;
244
+ const { dropped } = await this.mergeGroup(group, effective, ctx);
245
+ merged += dropped;
246
+ roundMerged += dropped;
247
+ for (const e of group)
248
+ consumed.add(e.id);
249
+ }
250
+ } while (roundMerged > 0);
251
+ }
252
+ return { merged, groups: dupGroups };
253
+ }
254
+ /**
255
+ * Rebuild the HNSW index from the current set of entries with embeddings.
256
+ * Returns a count snapshot + duration.
257
+ */
258
+ async compactHnsw() {
259
+ const adapter = this.service.getAdapter();
260
+ const entries = adapter.entries;
261
+ const index = adapter.index;
262
+ const before = index.size;
263
+ const t0 = Date.now();
264
+ // Build a fresh HNSW from current entries that have embeddings.
265
+ const cfg = index.getConfig();
266
+ const fresh = new HNSWIndex({
267
+ dimensions: cfg.dimensions,
268
+ M: cfg.M,
269
+ efConstruction: cfg.efConstruction,
270
+ maxElements: cfg.maxElements,
271
+ metric: cfg.metric,
272
+ });
273
+ for (const entry of entries.values()) {
274
+ if (entry.embedding) {
275
+ try {
276
+ await fresh.addPoint(entry.id, entry.embedding);
277
+ }
278
+ catch {
279
+ // Tolerate dimension mismatches in malformed datasets.
280
+ }
281
+ }
282
+ }
283
+ // Swap pointers atomically and re-forward events
284
+ adapter.index = fresh;
285
+ if (typeof adapter.emit === 'function') {
286
+ fresh.on('point:added', (data) => adapter.emit('index:added', data));
287
+ }
288
+ const durationMs = Date.now() - t0;
289
+ return { before, after: fresh.size, durationMs };
290
+ }
291
+ /**
292
+ * Sweep → dedup → compact. Used by the background timer and by the
293
+ * `nightlyLearner` AgentDB controller.
294
+ */
295
+ async runAll() {
296
+ const start = Date.now();
297
+ const sweep = await this.sweepExpired();
298
+ const dedup = await this.dedup();
299
+ const compact = await this.compactHnsw();
300
+ return {
301
+ sweep,
302
+ dedup,
303
+ compact,
304
+ totalDurationMs: Date.now() - start,
305
+ };
306
+ }
307
+ }
308
+ export default MemoryConsolidator;
309
+ //# sourceMappingURL=consolidator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"consolidator.js","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAuBlD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,8EAA8E;AAC9E,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B;;;;;;;;;;;;GAYG;AACH,MAAM,kBAAkB,GAAG,EAAE,CAAC;AA0C9B,MAAM,OAAO,kBAAkB;IAEV;IACA;IAFnB,YACmB,OAAoB,EACpB,OAA4B,EAAE;QAD9B,YAAO,GAAP,OAAO,CAAa;QACpB,SAAI,GAAJ,IAAI,CAA0B;IAC9C,CAAC;IAEJ;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAS,CAAC;QACjD,MAAM,OAAO,GAA6B,OAAO,CAAC,OAAO,CAAC;QAC1D,MAAM,cAAc,GAA6B,OAAO,CAAC,cAAc,CAAC;QACxE,MAAM,QAAQ,GAAwB,OAAO,CAAC,QAAQ,CAAC;QACvD,MAAM,QAAQ,GAA6B,OAAO,CAAC,QAAQ,CAAC;QAC5D,MAAM,KAAK,GAAc,OAAO,CAAC,KAAK,CAAC;QAEvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAkB,EAAE,CAAC;QACnC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;gBACrD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACzB,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACtD,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;YACnG,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,KAAK,CAAC,EAAE;gBAAE,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3E,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI;gBAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAClD,IAAI,OAAO;oBAAE,WAAW,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;YACvC,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;YAClC,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACL,OAAO,EAAE,QAAQ,CAAC,MAAM;YACxB,SAAS,EAAE,OAAO,CAAC,IAAI;YACvB,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,YAAY,CAAC,MAAqB,EAAE,QAAuB;QACjE,MAAM,MAAM,GACV,QAAQ,KAAK,aAAa;YACxB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACpE,CAAC,CAAC,6CAA6C;gBAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU,CACtB,MAAqB,EACrB,QAAuB,EACvB,GAOC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEnD,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;YAC5B,MAAM,CAAC,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,kDAAkD;YAClD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;oBACzD,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE;gBAAE,SAAS;YACjC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzB,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClD,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC/E,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACpC,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;gBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;gBAChB,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;YACD,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC;IAC1C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,KAAK,CAAC,QAAwB;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAS,CAAC;QACjD,MAAM,OAAO,GAA6B,OAAO,CAAC,OAAO,CAAC;QAC1D,MAAM,cAAc,GAA6B,OAAO,CAAC,cAAc,CAAC;QACxE,MAAM,QAAQ,GAAwB,OAAO,CAAC,QAAQ,CAAC;QACvD,MAAM,QAAQ,GAA6B,OAAO,CAAC,QAAQ,CAAC;QAC5D,MAAM,KAAK,GAAc,OAAO,CAAC,KAAK,CAAC;QACvC,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAEvH,MAAM,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;QAEvE,yDAAyD;QACzD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;QACjD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;gBAAE,SAAS;YACjC,SAAS,IAAI,CAAC,CAAC;YACf,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;YAClE,MAAM,IAAI,OAAO,CAAC;QACpB,CAAC;QAED,4DAA4D;QAC5D,EAAE;QACF,uDAAuD;QACvD,sEAAsE;QACtE,iEAAiE;QACjE,+DAA+D;QAC/D,qEAAqE;QACrE,gEAAgE;QAChE,oEAAoE;QACpE,uEAAuE;QACvE,oEAAoE;QACpE,sEAAsE;QACtE,6DAA6D;QAC7D,sEAAsE;QACtE,2BAA2B;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,4BAA4B,CAAC;QAChF,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC3D,IAAI,WAAmB,CAAC;YACxB,GAAG,CAAC;gBACF,WAAW,GAAG,CAAC,CAAC;gBAChB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;gBACnC,iEAAiE;gBACjE,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;oBAC1C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;wBACzE,SAAS;oBACX,CAAC;oBAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;oBACxF,MAAM,KAAK,GAAkB,CAAC,KAAK,CAAC,CAAC;oBACrC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,IAAI,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;4BAAE,SAAS;wBAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACtC,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,SAAS;4BAAE,SAAS;wBACjD,MAAM,UAAU,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC;wBACpC,IAAI,UAAU,IAAI,SAAS;4BAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrD,CAAC;oBAED,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;wBACtB,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBACvB,SAAS;oBACX,CAAC;oBAED,SAAS,IAAI,CAAC,CAAC;oBACf,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;oBACjE,MAAM,IAAI,OAAO,CAAC;oBAClB,WAAW,IAAI,OAAO,CAAC;oBACvB,KAAK,MAAM,CAAC,IAAI,KAAK;wBAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC,QAAQ,WAAW,GAAG,CAAC,EAAE;QAC5B,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAS,CAAC;QACjD,MAAM,OAAO,GAA6B,OAAO,CAAC,OAAO,CAAC;QAC1D,MAAM,KAAK,GAAc,OAAO,CAAC,KAAK,CAAC;QAEvC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;QAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEtB,gEAAgE;QAChE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC;YAC1B,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,CAAC,EAAE,GAAG,CAAC,CAAC;YACR,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;SACnB,CAAC,CAAC;QAEH,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;gBAClD,CAAC;gBAAC,MAAM,CAAC;oBACP,uDAAuD;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,iDAAiD;QACjD,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QACtB,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvC,KAAK,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,IAAS,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QACnC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO;YACL,KAAK;YACL,KAAK;YACL,OAAO;YACP,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SACpC,CAAC;IACJ,CAAC;CACF;AAED,eAAe,kBAAkB,CAAC"}
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Phase 4 — MemoryConsolidator (ADR-125)
3
+ *
4
+ * Verifies:
5
+ * - `sweepExpired()` after 1000 entries with `expiresAt` in the past →
6
+ * `entries.size === 0` AND `hnsw.size === 0` (Acceptance Criterion #4).
7
+ * - `dedup('keep-newest')` collapses content-hash duplicates and keeps the
8
+ * newest by `updatedAt`.
9
+ * - `dedup('keep-oldest')` keeps the oldest by `createdAt`.
10
+ * - `dedup('merge-tags')` unions tags and keeps the newest.
11
+ * - `compactHnsw` rebuilds the index so size matches `entries.size`.
12
+ * - Auto-run timer with a small interval calls `runAll()` at least once.
13
+ * - `nightlyLearner` controller delegates to `MemoryConsolidator.runAll()`
14
+ * when a `MemoryService` is registered.
15
+ */
16
+ export {};
17
+ //# sourceMappingURL=consolidator.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"consolidator.test.d.ts","sourceRoot":"","sources":["../src/consolidator.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}