agent-working-memory 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +118 -19
  2. package/dist/adapters/claude-code.d.ts.map +1 -1
  3. package/dist/adapters/claude-code.js +63 -3
  4. package/dist/adapters/claude-code.js.map +1 -1
  5. package/dist/adapters/common.d.ts.map +1 -1
  6. package/dist/adapters/common.js +329 -302
  7. package/dist/adapters/common.js.map +1 -1
  8. package/dist/api/routes.d.ts.map +1 -1
  9. package/dist/api/routes.js +31 -8
  10. package/dist/api/routes.js.map +1 -1
  11. package/dist/cli/migrate.js +29 -29
  12. package/dist/cli.js +82 -2
  13. package/dist/cli.js.map +1 -1
  14. package/dist/coordination/circuit-breaker.js +23 -23
  15. package/dist/coordination/routes.d.ts.map +1 -1
  16. package/dist/coordination/routes.js +174 -170
  17. package/dist/coordination/routes.js.map +1 -1
  18. package/dist/core/embeddings.d.ts.map +1 -1
  19. package/dist/core/embeddings.js +3 -0
  20. package/dist/core/embeddings.js.map +1 -1
  21. package/dist/core/entity-extract.d.ts +3 -0
  22. package/dist/core/entity-extract.d.ts.map +1 -0
  23. package/dist/core/entity-extract.js +47 -0
  24. package/dist/core/entity-extract.js.map +1 -0
  25. package/dist/core/salience.d.ts.map +1 -1
  26. package/dist/core/salience.js +14 -2
  27. package/dist/core/salience.js.map +1 -1
  28. package/dist/core/whoami.d.ts +24 -0
  29. package/dist/core/whoami.d.ts.map +1 -0
  30. package/dist/core/whoami.js +66 -0
  31. package/dist/core/whoami.js.map +1 -0
  32. package/dist/core/write-pipeline.d.ts +9 -0
  33. package/dist/core/write-pipeline.d.ts.map +1 -1
  34. package/dist/core/write-pipeline.js +109 -68
  35. package/dist/core/write-pipeline.js.map +1 -1
  36. package/dist/core/write-telemetry.d.ts +33 -0
  37. package/dist/core/write-telemetry.d.ts.map +1 -0
  38. package/dist/core/write-telemetry.js +110 -0
  39. package/dist/core/write-telemetry.js.map +1 -0
  40. package/dist/engine/activation.d.ts +22 -12
  41. package/dist/engine/activation.d.ts.map +1 -1
  42. package/dist/engine/activation.js +133 -17
  43. package/dist/engine/activation.js.map +1 -1
  44. package/dist/engine/consolidation-scheduler.d.ts +1 -1
  45. package/dist/engine/consolidation-scheduler.js +1 -1
  46. package/dist/engine/consolidation.d.ts +1 -0
  47. package/dist/engine/consolidation.d.ts.map +1 -1
  48. package/dist/engine/consolidation.js +18 -0
  49. package/dist/engine/consolidation.js.map +1 -1
  50. package/dist/engine/eval.d.ts.map +1 -1
  51. package/dist/engine/eval.js +5 -1
  52. package/dist/engine/eval.js.map +1 -1
  53. package/dist/index.js +21 -2
  54. package/dist/index.js.map +1 -1
  55. package/dist/mcp.d.ts +2 -1
  56. package/dist/mcp.d.ts.map +1 -1
  57. package/dist/mcp.js +212 -97
  58. package/dist/mcp.js.map +1 -1
  59. package/dist/onboard/index.d.ts +68 -0
  60. package/dist/onboard/index.d.ts.map +1 -0
  61. package/dist/onboard/index.js +265 -0
  62. package/dist/onboard/index.js.map +1 -0
  63. package/dist/recipes/index.d.ts +57 -0
  64. package/dist/recipes/index.d.ts.map +1 -0
  65. package/dist/recipes/index.js +81 -0
  66. package/dist/recipes/index.js.map +1 -0
  67. package/dist/storage/pglite-schema.d.ts.map +1 -1
  68. package/dist/storage/pglite-schema.js +170 -143
  69. package/dist/storage/pglite-schema.js.map +1 -1
  70. package/dist/storage/pglite.d.ts +5 -0
  71. package/dist/storage/pglite.d.ts.map +1 -1
  72. package/dist/storage/pglite.js +180 -138
  73. package/dist/storage/pglite.js.map +1 -1
  74. package/dist/storage/postgres.d.ts +5 -0
  75. package/dist/storage/postgres.d.ts.map +1 -1
  76. package/dist/storage/postgres.js +44 -2
  77. package/dist/storage/postgres.js.map +1 -1
  78. package/dist/storage/sqlite.d.ts +9 -0
  79. package/dist/storage/sqlite.d.ts.map +1 -1
  80. package/dist/storage/sqlite.js +394 -326
  81. package/dist/storage/sqlite.js.map +1 -1
  82. package/dist/types/engram.d.ts +14 -0
  83. package/dist/types/engram.d.ts.map +1 -1
  84. package/dist/types/engram.js.map +1 -1
  85. package/dist/version.d.ts +2 -0
  86. package/dist/version.d.ts.map +1 -0
  87. package/dist/version.js +27 -0
  88. package/dist/version.js.map +1 -0
  89. package/package.json +9 -1
  90. package/src/adapters/claude-code.ts +66 -3
  91. package/src/adapters/common.ts +538 -511
  92. package/src/api/index.ts +3 -3
  93. package/src/api/routes.ts +999 -970
  94. package/src/cli/migrate.ts +307 -307
  95. package/src/cli.ts +77 -2
  96. package/src/coordination/circuit-breaker.ts +83 -83
  97. package/src/coordination/failure-modes.ts +50 -50
  98. package/src/coordination/routes.ts +2155 -2150
  99. package/src/core/decay.ts +63 -63
  100. package/src/core/embeddings.ts +113 -110
  101. package/src/core/entity-extract.ts +47 -0
  102. package/src/core/index.ts +5 -5
  103. package/src/core/logger.ts +36 -36
  104. package/src/core/ml-worker-entry.ts +194 -194
  105. package/src/core/ml-worker.ts +281 -281
  106. package/src/core/query-expander.ts +122 -122
  107. package/src/core/reranker.ts +119 -119
  108. package/src/core/salience.ts +529 -514
  109. package/src/core/whoami.ts +92 -0
  110. package/src/core/write-pipeline.ts +60 -8
  111. package/src/core/write-telemetry.ts +131 -0
  112. package/src/engine/activation.ts +1468 -1369
  113. package/src/engine/confidence.ts +120 -120
  114. package/src/engine/consolidation-scheduler.ts +242 -242
  115. package/src/engine/consolidation.ts +887 -869
  116. package/src/engine/eval.ts +107 -102
  117. package/src/engine/eviction.ts +101 -101
  118. package/src/engine/index.ts +8 -8
  119. package/src/engine/retraction.ts +366 -366
  120. package/src/engine/staging.ts +74 -74
  121. package/src/index.ts +248 -226
  122. package/src/mcp.ts +1341 -1211
  123. package/src/onboard/index.ts +298 -0
  124. package/src/recipes/index.ts +125 -0
  125. package/src/storage/index.ts +3 -3
  126. package/src/storage/pglite-schema.ts +193 -166
  127. package/src/storage/pglite.ts +1420 -1372
  128. package/src/storage/postgres.ts +50 -2
  129. package/src/storage/sqlite.ts +1936 -1861
  130. package/src/storage/store.ts +80 -80
  131. package/src/types/agent.ts +67 -67
  132. package/src/types/checkpoint.ts +46 -46
  133. package/src/types/engram.ts +22 -0
  134. package/src/types/eval.ts +100 -100
  135. package/src/types/index.ts +6 -6
  136. package/src/version.ts +26 -0
@@ -1,869 +1,887 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * Sleep Cycle — offline memory consolidation.
5
- *
6
- * Models the brain's consolidation during sleep:
7
- * 1. Replay — find clusters of semantically similar memories
8
- * 2. Strengthen — reinforce edges within clusters + access-weighted boost
9
- * 3. Bridge — create cross-cluster shortcuts between related topic areas
10
- * 4. Decay — weaken unused edges, prune dead ones
11
- * 5. Homeostasis — normalize outgoing edge weights to prevent hub explosion
12
- * 6. Forget — archive/delete memories that were never retrieved (age-gated)
13
- * 7. Sweep — promote or discard uncertain (staging) memories
14
- *
15
- * No artificial "summary nodes" are created. Instead, the associative
16
- * graph gets denser where knowledge overlaps and sparser where it doesn't.
17
- * The beam search graph walk in activation.ts naturally propagates through
18
- * these strengthened pathways.
19
- *
20
- * Run between sessions or on a timer (e.g., every few hours).
21
- */
22
-
23
- import { cosineSimilarity } from '../core/embeddings.js';
24
- import { strengthenAssociation, decayAssociation } from '../core/hebbian.js';
25
- import type { Engram } from '../types/index.js';
26
- import type { IEngramStore as EngramStore } from '../storage/store.js';
27
- import type { ConnectionEngine } from './connections.js';
28
-
29
- /** Cosine similarity for initial candidate detection (single-link entry gate) */
30
- const SIMILARITY_THRESHOLD = 0.65;
31
-
32
- /** Minimum pairwise cosine for cluster diameter enforcement.
33
- * Prevents chaining: a candidate must be this similar to ALL cluster members. */
34
- const MIN_PAIRWISE_COS = 0.50;
35
-
36
- /** Lower threshold for cross-cluster bridge edges */
37
- const BRIDGE_THRESHOLD = 0.25;
38
-
39
- /** Minimum edge weight to form a new connection during replay */
40
- const INITIAL_EDGE_WEIGHT = 0.3;
41
-
42
- /** Boost factor for strengthening existing edges between cluster members */
43
- const CONSOLIDATION_SIGNAL = 0.5;
44
-
45
- /** Max new edges to create per sleep cycle (prevent graph explosion) */
46
- const MAX_NEW_EDGES_PER_CYCLE = 50;
47
-
48
- /** Max bridge edges per cycle (cross-cluster shortcuts) */
49
- const MAX_BRIDGE_EDGES_PER_CYCLE = 20;
50
-
51
- /** Edge weight below which we prune during decay */
52
- const PRUNE_THRESHOLD = 0.01;
53
-
54
- /** Target total outgoing edge weight per node (homeostasis) */
55
- const HOMEOSTASIS_TARGET = 10.0;
56
-
57
- /** Grace period before forgetting curve starts (days) */
58
- const FORGET_GRACE_DAYS = 7;
59
-
60
- /** Consolidation cycles before 0-access memories get archived */
61
- const FORGET_CYCLE_THRESHOLD = 5;
62
-
63
- /** Percentile of edge count distribution used for forgetting protection (0-1) */
64
- const EDGE_PROTECTION_PERCENTILE = 0.25;
65
-
66
- /** Age at which never-retrieved memories get archived (days) */
67
- const FORGET_ARCHIVE_DAYS = 30;
68
-
69
- /** Age at which archived, never-retrieved, unconnected memories get deleted (days) */
70
- const FORGET_DELETE_DAYS = 90;
71
-
72
- /** Cosine similarity above which two low-confidence memories are considered redundant.
73
- * MiniLM-L6 paraphrases typically score 0.75-0.88 cosine; 0.85 misses most of them.
74
- * 0.75 catches paraphrases while keeping precision above 0.60 for unrelated facts. */
75
- const REDUNDANCY_THRESHOLD = 0.75;
76
-
77
- /** Max redundant memories to prune per cycle (gradual, not sudden).
78
- * Raised from 10 to 25 the eval harness runs multiple cycles anyway,
79
- * but faster convergence reduces consolidation time for larger pools. */
80
- const MAX_REDUNDANCY_PRUNE_PER_CYCLE = 25;
81
-
82
- /** Max confidence drift per consolidation cycle (prevents runaway) */
83
- const CONFIDENCE_DRIFT_CAP = 0.03;
84
-
85
- /**
86
- * Content fade — Paper 1 (PLOS Comp Biology, storage degradation).
87
- * Memories whose content has not been recalled in a long time get coarsened:
88
- * the content string is trimmed to FADE_KEEP_CHARS with a "[faded]" marker,
89
- * while concept, tags, and embedding are preserved.
90
- *
91
- * Rationale: human memory degrades the detailed surface but retains the gist
92
- * and the cue-association pathway. The faded engram still surfaces by tag/concept
93
- * BM25 and by vector similarity (embedding preserved), but the full content body
94
- * is gone which is the correct behavior for a memory that has not been useful
95
- * enough to rehearse in a long time.
96
- *
97
- * Fade comes BEFORE archive in the lifecycle (active → fading → archived → deleted),
98
- * applied only to engrams that have been accessed at least once. Engrams that have
99
- * never been accessed go straight to archive via the standard forgetting path
100
- * they have no surface worth preserving.
101
- *
102
- * Env: AWM_FADE_DAYS_SINCE_ACCESS (default 45),
103
- * AWM_FADE_KEEP_CHARS (default 150),
104
- * AWM_FADE_MIN_CONTENT_LEN (default 250),
105
- * AWM_FADE_MAX_PER_CYCLE (default 25).
106
- */
107
- const FADE_DAYS_SINCE_ACCESS = Number(process.env.AWM_FADE_DAYS_SINCE_ACCESS ?? 45);
108
- const FADE_KEEP_CHARS = Number(process.env.AWM_FADE_KEEP_CHARS ?? 150);
109
- const FADE_MIN_CONTENT_LEN = Number(process.env.AWM_FADE_MIN_CONTENT_LEN ?? 250);
110
- const FADE_MAX_PER_CYCLE = Number(process.env.AWM_FADE_MAX_PER_CYCLE ?? 25);
111
- const FADE_PROTECTED_ACCESS_COUNT = 10;
112
- const FADE_MARKER = '… [faded]';
113
- const FADE_PROTECTED_CLASSES = new Set(['canonical', 'structural']);
114
-
115
- /** Days without recall before confidence starts drifting down */
116
- const CONFIDENCE_NEGLECT_DAYS = 30;
117
-
118
- export interface ConsolidationResult {
119
- clustersFound: number;
120
- edgesStrengthened: number;
121
- edgesCreated: number;
122
- bridgesCreated: number;
123
- edgesDecayed: number;
124
- edgesPruned: number;
125
- edgesNormalized: number;
126
- memoriesForgotten: number;
127
- memoriesArchived: number;
128
- memoriesFaded: number;
129
- redundancyPruned: number;
130
- confidenceAdjusted: number;
131
- stagingPromoted: number;
132
- stagingDiscarded: number;
133
- engramsProcessed: number;
134
- synthesesCreated: number;
135
- }
136
-
137
- const MAX_SYNTHESES_PER_CYCLE = 5;
138
- const MIN_CLUSTER_SIZE_FOR_SYNTHESIS = 3;
139
-
140
- /** Shared stopwords for synthesis keyword extraction */
141
- const SYNTH_STOPWORDS = new Set(['the', 'is', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'for',
142
- 'on', 'with', 'that', 'this', 'it', 'was', 'are', 'be', 'has', 'had', 'but', 'not', 'from',
143
- 'by', 'as', 'at', 'i', 'you', 'we', 'my', 'your', 'can', 'will', 'do', 'did', 'if', 'user',
144
- 'assistant', 'would', 'like', 'just', 'also', 'about', 'really', 'think', 'know', 'want',
145
- 'here', 'there', 'some', 'more', 'very', 'been', 'have', 'what', 'when', 'how', 'they']);
146
-
147
- export class ConsolidationEngine {
148
- private store: EngramStore;
149
- private connectionEngine: ConnectionEngine | null;
150
-
151
- /**
152
- * Optional `connectionEngine` is drained at the start of each
153
- * consolidation pass (v0.8.2). Connection discovery is no longer
154
- * triggered per write the work batches into the existing sleep cycle
155
- * so writes return in ~50 ms instead of being pegged by an inline
156
- * embed + rerank cycle (~200-500 ms/write under load).
157
- */
158
- constructor(store: EngramStore, connectionEngine?: ConnectionEngine) {
159
- this.store = store;
160
- this.connectionEngine = connectionEngine ?? null;
161
- }
162
-
163
- /**
164
- * Run a full sleep cycle for an agent.
165
- *
166
- * Phase 0: Connection drain — discover associations for engrams enqueued
167
- * by writes since the last consolidation (v0.8.2)
168
- * Phase 1: Replay find clusters of semantically similar memories
169
- * Phase 2: Strengthenreinforce edges within clusters (access-weighted)
170
- * Phase 3: Bridgecreate cross-cluster shortcuts
171
- * Phase 4: Decayweaken unused edges, prune dead ones
172
- * Phase 5: Homeostasisnormalize outgoing edge weights per node
173
- * Phase 5.5: Content fade coarsen un-recalled engrams (Paper 1)
174
- * Phase 6: Forgetarchive/delete memories never retrieved (age-gated)
175
- * Phase 6.5: Redundancy prune
176
- * Phase 6.7: Confidence drift — adjust confidence based on structural signals
177
- * Phase 7: Sweepcheck staging buffer for resonance
178
- */
179
- async consolidate(agentId: string): Promise<ConsolidationResult> {
180
- // --- Phase 0: Connection drain ---
181
- // Discover associations for engrams enqueued by writes since the last
182
- // consolidation. Per-write inline discovery was removed in v0.8.2
183
- // because each findConnections() call runs a full activation cycle
184
- // (embed + rerank, ~200-500 ms event-loop block per write).
185
- if (this.connectionEngine) {
186
- try { await this.connectionEngine.processQueue(); } catch { /* best-effort */ }
187
- }
188
- const result: ConsolidationResult = {
189
- clustersFound: 0,
190
- edgesStrengthened: 0,
191
- edgesCreated: 0,
192
- bridgesCreated: 0,
193
- edgesDecayed: 0,
194
- edgesPruned: 0,
195
- edgesNormalized: 0,
196
- memoriesForgotten: 0,
197
- memoriesArchived: 0,
198
- memoriesFaded: 0,
199
- redundancyPruned: 0,
200
- confidenceAdjusted: 0,
201
- stagingPromoted: 0,
202
- stagingDiscarded: 0,
203
- engramsProcessed: 0,
204
- synthesesCreated: 0,
205
- };
206
-
207
- // --- Phase 1: Replay ---
208
- // Get all active engrams, backfill missing embeddings
209
- const allActive = await this.store.getEngramsByAgent(agentId, 'active');
210
- const needsEmbedding = allActive.filter(e => !e.embedding || e.embedding.length === 0);
211
- if (needsEmbedding.length > 0) {
212
- try {
213
- const { embedBatch, getModelId } = await import('../core/embeddings.js');
214
- const BATCH_SIZE = 32;
215
- const modelId = getModelId?.() ?? 'unknown';
216
- for (let b = 0; b < needsEmbedding.length; b += BATCH_SIZE) {
217
- const batch = needsEmbedding.slice(b, b + BATCH_SIZE);
218
- try {
219
- const texts = batch.map(e => `${e.concept} ${e.content}`);
220
- const vecs = await embedBatch(texts);
221
- for (let j = 0; j < batch.length; j++) {
222
- await this.store.updateEmbedding(batch[j].id, vecs[j], modelId);
223
- batch[j].embedding = vecs[j];
224
- }
225
- } catch { /* batch failed, skip — non-fatal */ }
226
- }
227
- if (needsEmbedding.length > 0) {
228
- console.log(`[consolidation] Backfilled ${needsEmbedding.filter(e => e.embedding?.length).length}/${needsEmbedding.length} embeddings (model: ${modelId})`);
229
- }
230
- } catch { /* embeddings module unavailable */ }
231
- }
232
- const engrams = allActive.filter(e => e.embedding && e.embedding.length > 0);
233
-
234
- result.engramsProcessed = engrams.length;
235
- // Clustering requires at least two engrams to form pairs. Per-engram phases
236
- // (fade, forget, confidence drift, staging sweep) still run for singletons —
237
- // they don't need a graph.
238
- const clusters = engrams.length >= 2 ? this.findClusters(engrams) : [];
239
- result.clustersFound = clusters.length;
240
-
241
- // --- Phase 2: Strengthen (access-weighted) ---
242
- // Memories that are retrieved more often get stronger consolidation.
243
- // This mirrors how the brain preferentially consolidates practiced memories.
244
- let newEdges = 0;
245
- for (const cluster of clusters) {
246
- for (let i = 0; i < cluster.length; i++) {
247
- for (let j = i + 1; j < cluster.length; j++) {
248
- const a = cluster[i];
249
- const b = cluster[j];
250
-
251
- // Access-weighted signal: more retrieved = stronger consolidation
252
- const accessFactor = Math.min(
253
- 1.0,
254
- 0.3 + 0.7 * Math.log1p(a.accessCount + b.accessCount) / Math.log1p(20),
255
- );
256
-
257
- const existing = await this.store.getAssociation(a.id, b.id);
258
- if (existing) {
259
- const newWeight = strengthenAssociation(
260
- existing.weight, CONSOLIDATION_SIGNAL * accessFactor, 0.25,
261
- );
262
- await this.store.upsertAssociation(
263
- a.id, b.id, newWeight, existing.type, existing.confidence,
264
- );
265
- result.edgesStrengthened++;
266
- } else if (newEdges < MAX_NEW_EDGES_PER_CYCLE) {
267
- await this.store.upsertAssociation(
268
- a.id, b.id, INITIAL_EDGE_WEIGHT * accessFactor, 'connection',
269
- );
270
- newEdges++;
271
- result.edgesCreated++;
272
- }
273
- }
274
- }
275
- }
276
-
277
- // --- Phase 2.5: Two types of synthesis ---
278
- //
279
- // Type A: SESSION SYNTHESIS (perfect recall)
280
- // Groups by shared metadata tags (sid=, proj=, topic=).
281
- // Summarizes what happened in a conversation/project session.
282
- // Helps find specific facts by providing a topical anchor.
283
- //
284
- // Type B: PATTERN SYNTHESIS (novel recall)
285
- // Uses the existing vector-similarity clusters.
286
- // Finds structural patterns across disparate topics.
287
- // "Debugging X by Y" + "Resolving A by B" → pattern: "conflict → ordering"
288
- // Lower confidence — these are speculative connections, not facts.
289
-
290
- let synthCount = 0;
291
-
292
- // --- Type A: Session synthesis (tag-based grouping) ---
293
- // Group engrams by shared session/project tags, NOT vector similarity
294
- const tagGroups = new Map<string, Engram[]>();
295
- for (const e of engrams) {
296
- if (e.tags.includes('synth=true')) continue; // Skip existing syntheses
297
- for (const tag of e.tags) {
298
- if (tag.startsWith('sid=') || tag.startsWith('proj=') || tag.startsWith('topic=')) {
299
- const group = tagGroups.get(tag) ?? [];
300
- group.push(e);
301
- tagGroups.set(tag, group);
302
- }
303
- }
304
- }
305
-
306
- for (const [tag, group] of tagGroups) {
307
- if (group.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
308
- if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
309
-
310
- // Check if a synthesis for this tag already exists
311
- const existing = engrams.find(e =>
312
- e.tags.includes('synth=true') && e.tags.includes(tag)
313
- );
314
- if (existing) continue;
315
-
316
- // Extract key terms from this group
317
- const wordCounts = new Map<string, number>();
318
- for (const e of group) {
319
- const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
320
- for (const w of words) {
321
- if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
322
- wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
323
- }
324
- }
325
- }
326
- const keyTerms = [...wordCounts.entries()]
327
- .sort((a, b) => b[1] - a[1])
328
- .slice(0, 15)
329
- .map(([word]) => word);
330
-
331
- // Extract unique concepts (deduplicated)
332
- const concepts = [...new Set(group.map(e => e.concept))];
333
-
334
- const synthContent = [
335
- `Session summary (${tag}, ${group.length} turns).`,
336
- `Key topics: ${keyTerms.slice(0, 8).join(', ')}.`,
337
- `Discussed: ${keyTerms.slice(8).join(', ')}.`,
338
- ].join(' ');
339
-
340
- const synthEngram = await this.store.createEngram({
341
- agentId,
342
- concept: `session: ${tag} (${keyTerms.slice(0, 3).join(', ')})`,
343
- content: synthContent,
344
- tags: [tag, 'synth=true', 'synth-type=session'],
345
- salience: 0.6,
346
- confidence: 0.55,
347
- memoryType: 'semantic',
348
- });
349
-
350
- // Set embedding to group centroid
351
- const centroid = this.computeCentroid(group);
352
- if (centroid.length > 0) {
353
- await this.store.updateEmbedding(synthEngram.id, centroid);
354
- }
355
-
356
- // Link to sources
357
- for (const source of group.slice(0, 10)) { // Cap links to prevent explosion
358
- await this.store.upsertAssociation(synthEngram.id, source.id, 0.4, 'causal');
359
- }
360
-
361
- synthCount++;
362
- result.synthesesCreated++;
363
- }
364
-
365
- // --- Type B: Pattern synthesis (vector-similarity clusters, speculative) ---
366
- // Only create these for clusters where members come from DIFFERENT sessions/projects.
367
- // This finds cross-domain patterns: "debugging technique A" + "architecture pattern B"
368
- for (const cluster of clusters) {
369
- if (cluster.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
370
- if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
371
- if (cluster.some(e => e.tags.includes('synth=true'))) continue;
372
-
373
- // Only create pattern synthesis if cluster spans multiple sessions
374
- const sessionTags = new Set<string>();
375
- for (const e of cluster) {
376
- for (const tag of e.tags) {
377
- if (tag.startsWith('sid=') || tag.startsWith('proj=')) sessionTags.add(tag);
378
- }
379
- }
380
- if (sessionTags.size < 2) continue; // Same session → skip (Type A handles it)
381
-
382
- const wordCounts = new Map<string, number>();
383
- for (const e of cluster) {
384
- const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
385
- for (const w of words) {
386
- if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
387
- wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
388
- }
389
- }
390
- }
391
- const keyTerms = [...wordCounts.entries()]
392
- .sort((a, b) => b[1] - a[1])
393
- .slice(0, 10)
394
- .map(([word]) => word);
395
-
396
- const synthContent = [
397
- `Pattern across ${sessionTags.size} sessions (${cluster.length} memories).`,
398
- `Common themes: ${keyTerms.join(', ')}.`,
399
- ].join(' ');
400
-
401
- const synthEngram = await this.store.createEngram({
402
- agentId,
403
- concept: `pattern: ${keyTerms.slice(0, 3).join(', ')}`,
404
- content: synthContent,
405
- tags: [...sessionTags, 'synth=true', 'synth-type=pattern'],
406
- salience: 0.5, // Lower — speculative
407
- confidence: 0.4, // Lower — these are hypotheses not facts
408
- memoryType: 'semantic',
409
- });
410
-
411
- const centroid = this.computeCentroid(cluster);
412
- if (centroid.length > 0) {
413
- await this.store.updateEmbedding(synthEngram.id, centroid);
414
- }
415
-
416
- for (const source of cluster.slice(0, 8)) {
417
- await this.store.upsertAssociation(synthEngram.id, source.id, 0.3, 'bridge');
418
- }
419
-
420
- synthCount++;
421
- result.synthesesCreated++;
422
- }
423
-
424
- // --- Phase 3: Direct cross-cluster bridging ---
425
- // Find the closest pair of memories between each cluster pair and bridge them.
426
- if (clusters.length >= 2) {
427
- const MIN_BRIDGE_SIM = 0.15;
428
- let bridges = 0;
429
- for (let i = 0; i < clusters.length && bridges < MAX_BRIDGE_EDGES_PER_CYCLE; i++) {
430
- for (let j = i + 1; j < clusters.length && bridges < MAX_BRIDGE_EDGES_PER_CYCLE; j++) {
431
- let bestSim = -1;
432
- let bestA: Engram | null = null;
433
- let bestB: Engram | null = null;
434
- for (const a of clusters[i]) {
435
- if (!a.embedding) continue;
436
- for (const b of clusters[j]) {
437
- if (!b.embedding) continue;
438
- const s = cosineSimilarity(a.embedding, b.embedding);
439
- if (s > bestSim) { bestSim = s; bestA = a; bestB = b; }
440
- }
441
- }
442
- if (bestA && bestB && bestSim > MIN_BRIDGE_SIM) {
443
- const existing = await this.store.getAssociation(bestA.id, bestB.id);
444
- if (!existing) {
445
- await this.store.upsertAssociation(bestA.id, bestB.id, bestSim, 'bridge');
446
- await this.store.upsertAssociation(bestB.id, bestA.id, bestSim, 'bridge');
447
- bridges++;
448
- result.bridgesCreated++;
449
- }
450
- }
451
- }
452
- }
453
- }
454
-
455
- // --- Phase 4: Decay (confidence-modulated) ---
456
- // High-confidence edges decay slower. This means edges between memories
457
- // that received positive feedback are more durable just like how
458
- // practiced memories are more resistant to forgetting in the brain.
459
- // Base half-life: 7 days. High-confidence (0.8+) gets up to 30 days.
460
- const engramMap = new Map(engrams.map(e => [e.id, e]));
461
- const associations = await this.store.getAllAssociations(agentId);
462
- for (const assoc of associations) {
463
- const daysSince =
464
- (Date.now() - assoc.lastActivated.getTime()) / (1000 * 60 * 60 * 24);
465
- if (daysSince < 0.5) continue;
466
-
467
- const fromEngram = engramMap.get(assoc.fromEngramId);
468
- const toEngram = engramMap.get(assoc.toEngramId);
469
- const fromConf = fromEngram?.confidence ?? 0.5;
470
- const toConf = toEngram?.confidence ?? 0.5;
471
- const maxConf = Math.max(fromConf, toConf);
472
- const maxAccess = Math.max(fromEngram?.accessCount ?? 0, toEngram?.accessCount ?? 0);
473
- const accessBoost = Math.min(2.0, 1.0 + 0.5 * Math.log1p(maxAccess));
474
- const halfLifeDays = Math.min(
475
- 7 * (1 + 2 * Math.max(0, (maxConf - 0.5) / 0.5)) * accessBoost,
476
- 42 // Hard cap: 6 weeks max
477
- );
478
-
479
- const newWeight = decayAssociation(assoc.weight, daysSince, halfLifeDays);
480
- if (newWeight < PRUNE_THRESHOLD) {
481
- await this.store.deleteAssociation(assoc.id);
482
- result.edgesPruned++;
483
- } else if (Math.abs(newWeight - assoc.weight) > 0.001) {
484
- await this.store.upsertAssociation(
485
- assoc.fromEngramId, assoc.toEngramId,
486
- newWeight, assoc.type, assoc.confidence,
487
- );
488
- result.edgesDecayed++;
489
- }
490
- }
491
-
492
- // --- Phase 5: Synaptic homeostasis ---
493
- // Normalize total outgoing edge weight per node to prevent hub explosion.
494
- // Nodes with many strong edges get scaled down so relative weights stay meaningful.
495
- const engramIds = new Set(engrams.map(e => e.id));
496
- for (const id of engramIds) {
497
- const outgoing = await this.store.getOutgoingAssociations(id);
498
- const totalWeight = outgoing.reduce((sum, a) => sum + a.weight, 0);
499
- if (totalWeight > HOMEOSTASIS_TARGET) {
500
- const scale = HOMEOSTASIS_TARGET / totalWeight;
501
- for (const edge of outgoing) {
502
- const newWeight = edge.weight * scale;
503
- if (newWeight < PRUNE_THRESHOLD) {
504
- await this.store.deleteAssociation(edge.id);
505
- result.edgesPruned++;
506
- } else {
507
- await this.store.upsertAssociation(
508
- edge.fromEngramId, edge.toEngramId,
509
- newWeight, edge.type, edge.confidence,
510
- );
511
- }
512
- }
513
- result.edgesNormalized++;
514
- }
515
- }
516
-
517
- // --- Phase 5.5: Content fade (Paper 1: storage degradation) ---
518
- // Coarsens engrams that have been accessed before but have gone stale.
519
- // Trims content surface area while preserving concept, tags, and embedding so
520
- // the engram still participates in BM25 + vector recall — just with less
521
- // body to score against. This models how human memory loses surface detail
522
- // while retaining cue-association pathways.
523
- //
524
- // Fade applies only to accessed engrams (the never-accessed ones go straight
525
- // to archive via Phase 6). Heavily-used (accessCount >= FADE_PROTECTED_ACCESS_COUNT),
526
- // canonical, structural, and short-content engrams are excluded.
527
- let fadeCount = 0;
528
- for (const engram of engrams) {
529
- if (fadeCount >= FADE_MAX_PER_CYCLE) break;
530
- if (engram.accessCount === 0) continue;
531
- if (engram.accessCount >= FADE_PROTECTED_ACCESS_COUNT) continue;
532
- if (FADE_PROTECTED_CLASSES.has(engram.memoryClass)) continue;
533
- if (engram.retracted) continue;
534
- if (engram.content.length < FADE_MIN_CONTENT_LEN) continue;
535
- if (engram.content.endsWith(FADE_MARKER)) continue; // Already faded
536
- const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
537
- if (daysSinceAccess < FADE_DAYS_SINCE_ACCESS) continue;
538
-
539
- const trimmed = engram.content.slice(0, FADE_KEEP_CHARS).trimEnd() + FADE_MARKER;
540
- await this.store.updateContent(engram.id, trimmed);
541
- await this.store.updateStage(engram.id, 'fading');
542
- engram.content = trimmed;
543
- engram.stage = 'fading';
544
- result.memoriesFaded++;
545
- fadeCount++;
546
- }
547
-
548
- // --- Phase 6: Forgetting (age-gated) ---
549
- // Models how human memory actually works:
550
- // - New memories get a grace period (too new to judge)
551
- // - Retrieval acts as rehearsal — resets the forgetting clock
552
- // - Well-connected memories persist (edges = integration into knowledge)
553
- // - Old, isolated, unretrieved memories fade to archive (not deleted)
554
- // - Archived memories can still be recovered via deep search
555
- // - Only truly orphaned, ancient memories get deleted
556
- //
557
- // Key insight: outdated memories still have value as historical context.
558
- // "We used to use X" helps explain why we now use Y.
559
- // Compute edge count percentile for relative protection threshold.
560
- // With avg 12 edges/node, an absolute threshold of 3 protects everything.
561
- // Use 25th percentile so "weakly connected" is relative to actual graph density.
562
- const edgeCounts = await Promise.all(engrams.map(async e => await this.store.countAssociationsFor(e.id)));
563
- edgeCounts.sort((a, b) => a - b);
564
- const percentileIdx = Math.floor(edgeCounts.length * EDGE_PROTECTION_PERCENTILE);
565
- const baseEdgeThreshold = edgeCounts.length > 0 ? edgeCounts[percentileIdx] : 3;
566
-
567
- // Get consolidation cycle count for cycle-based archiving
568
- const cycleCount = await this.store.getConsolidationCycleCount(agentId);
569
-
570
- for (const engram of engrams) {
571
- // Skip engrams that just transitioned to 'fading' in Phase 5.5 this cycle.
572
- // Fade is intended as an intermediate stage between active and archived —
573
- // the engram should live in 'fading' for at least one more recall window
574
- // before being eligible for archival. Without this guard, a same-cycle
575
- // fade-then-archive sequence collapses the active pool to zero (observed
576
- // in the stress test scale phase around cycle 90).
577
- if (engram.stage !== 'active') continue;
578
-
579
- const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
580
- if (ageDays < FORGET_GRACE_DAYS) continue; // Grace period — too new to judge
581
-
582
- const edgeCount = await this.store.countAssociationsFor(engram.id);
583
-
584
- // Use relative threshold (percentile-based) instead of absolute.
585
- // High-confidence memories need fewer edges to survive.
586
- const confReduction = engram.confidence > 0.7
587
- ? Math.min(0.6, (engram.confidence - 0.7) * 2)
588
- : 0;
589
- const edgeProtectionThreshold = Math.max(1, Math.round(baseEdgeThreshold * (1 - confReduction)));
590
- if (edgeCount > edgeProtectionThreshold) continue;
591
-
592
- // Cycle-based archive: 0-access memories archived after N cycles
593
- // regardless of age. Handles small pools where time thresholds are too generous.
594
- if (engram.accessCount === 0 && cycleCount >= FORGET_CYCLE_THRESHOLD) {
595
- await this.store.updateStage(engram.id, 'archived');
596
- result.memoriesArchived++;
597
- continue;
598
- }
599
-
600
- // Compute effective forgetting threshold based on memory strength signals.
601
- // Rehearsal (access + feedback) extends protection but NEVER makes immortal.
602
- // Models a sharp 20-year senior dev: confirmed knowledge persists for months/years.
603
- // - Base: FORGET_ARCHIVE_DAYS (30 days)
604
- // - Access extends by log-scaled factor: 5 accesses ≈ 2x, 10 ≈ 2.5x
605
- // - Confidence modulates up to 4x (0.5→1x, 0.7→2.6x, 0.8→3.4x, 1.0→4x)
606
- // - Hard cap: 12x base (360 days) — even the sharpest memory fades after a year
607
- const accessFactor = 1 + Math.log1p(engram.accessCount) * 0.6;
608
- const confFactor = 1 + 3 * Math.max(0, (engram.confidence - 0.5) / 0.5);
609
- const effectiveArchiveDays = Math.min(
610
- FORGET_ARCHIVE_DAYS * accessFactor * confFactor,
611
- FORGET_ARCHIVE_DAYS * 12, // Hard cap: 12x base (360 days)
612
- );
613
-
614
- const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
615
-
616
- if (engram.accessCount === 0 && ageDays > FORGET_ARCHIVE_DAYS) {
617
- // Never retrieved, old, weakly connected → archive
618
- await this.store.updateStage(engram.id, 'archived');
619
- result.memoriesArchived++;
620
- } else if (engram.accessCount > 0 && daysSinceAccess > effectiveArchiveDays) {
621
- // Accessed before but not recently enough given its strength — archive
622
- await this.store.updateStage(engram.id, 'archived');
623
- result.memoriesArchived++;
624
- }
625
- }
626
-
627
- // Check archived memories for deletion — only truly orphaned ancient ones
628
- const archived = await this.store.getEngramsByAgent(agentId, 'archived');
629
- for (const engram of archived) {
630
- const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
631
- const edgeCount = await this.store.countAssociationsFor(engram.id);
632
-
633
- if (engram.accessCount === 0 && ageDays > FORGET_DELETE_DAYS && edgeCount === 0) {
634
- // Very old, never accessed, completely isolated truly forgotten
635
- await this.store.deleteEngram(engram.id);
636
- result.memoriesForgotten++;
637
- }
638
- // Otherwise: stay archived still searchable, just not in active recall
639
- }
640
-
641
- // --- Phase 6.5: Redundancy pruning ---
642
- // A senior dev doesn't store 30 nearly-identical memories. When multiple
643
- // low-confidence memories are semantically redundant (cosine > 0.85), keep
644
- // only the one with highest accessCount + confidence and archive the rest.
645
- // This naturally defeats volume-based attacks (narcissistic interference,
646
- // spam) while improving signal-to-noise ratio for linked memories.
647
- // High-confidence memories (feedback-confirmed) are never pruned — they
648
- // represent verified knowledge worth keeping even if similar.
649
- // Only consider memories that are both low-confidence AND rarely accessed.
650
- // Memories retrieved 3+ times have proven useful — they stay even if similar
651
- // to others. This prevents pruning seed memories that match bulk templates.
652
- const lowConfEngrams = engrams.filter(e =>
653
- e.confidence < 0.6 && e.accessCount < 3 && e.embedding && e.embedding.length > 0);
654
- const pruned = new Set<string>();
655
- let redundancyCount = 0;
656
-
657
- // Sort by quality: highest accessCount + confidence first (survivors)
658
- const sortedLow = [...lowConfEngrams].sort((a, b) =>
659
- (b.accessCount + b.confidence * 10) - (a.accessCount + a.confidence * 10));
660
-
661
- for (let i = 0; i < sortedLow.length && redundancyCount < MAX_REDUNDANCY_PRUNE_PER_CYCLE; i++) {
662
- if (pruned.has(sortedLow[i].id)) continue;
663
- for (let j = i + 1; j < sortedLow.length && redundancyCount < MAX_REDUNDANCY_PRUNE_PER_CYCLE; j++) {
664
- if (pruned.has(sortedLow[j].id)) continue;
665
- if (!sortedLow[i].embedding || !sortedLow[j].embedding) continue;
666
-
667
- const sim = cosineSimilarity(sortedLow[i].embedding!, sortedLow[j].embedding!);
668
- if (sim >= REDUNDANCY_THRESHOLD) {
669
- const survivorId = sortedLow[i].id;
670
- const prunedId = sortedLow[j].id;
671
-
672
- // Transfer associations from pruned memory to survivor
673
- const prunedEdges = await this.store.getAssociationsFor(prunedId);
674
- for (const edge of prunedEdges) {
675
- const peerId = edge.fromEngramId === prunedId ? edge.toEngramId : edge.fromEngramId;
676
- if (peerId === survivorId) continue; // Skip self-loops
677
- await this.store.upsertAssociation(survivorId, peerId, edge.weight, edge.type, edge.confidence);
678
- }
679
-
680
- // Merge tags from pruned to survivor
681
- const survivor = sortedLow[i];
682
- const prunedMem = sortedLow[j];
683
- const mergedTags = [...new Set([...survivor.tags, ...prunedMem.tags])];
684
- if (mergedTags.length > survivor.tags.length) {
685
- await this.store.updateTags(survivorId, mergedTags);
686
- }
687
-
688
- // Archive the lower-quality duplicate
689
- await this.store.updateStage(prunedId, 'archived');
690
- pruned.add(prunedId);
691
- redundancyCount++;
692
- }
693
- }
694
- }
695
- result.redundancyPruned = redundancyCount;
696
-
697
- // --- Phase 6.7: Confidence drift ---
698
- // Adjust confidence based on structural signals that emerge from the graph.
699
- // This makes confidence evolve over time without explicit feedback calls.
700
- //
701
- // Three signals:
702
- // 1. Well-clustered memories (appeared in 1+ clusters) get a small boost
703
- // — they're integrated into the knowledge graph, likely valuable.
704
- // 2. Isolated memories (0 edges after consolidation) get a small penalty
705
- // — nothing connects to them, possibly noise.
706
- // 3. Neglected memories (not recalled in 30+ days) drift toward 0.3
707
- // — if the system never needs them, they're probably not important.
708
- //
709
- // All adjustments are capped at ±0.03 per cycle to prevent runaway.
710
- // Confidence is floored at 0.15 (never reaches 0 — retraction handles that).
711
- // Confidence is capped at 0.85 (only explicit feedback can push above).
712
- const clusteredIds = new Set<string>();
713
- for (const cluster of clusters) {
714
- for (const e of cluster) clusteredIds.add(e.id);
715
- }
716
-
717
- for (const engram of engrams) {
718
- let drift = 0;
719
- const edgeCount = await this.store.countAssociationsFor(engram.id);
720
- const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
721
-
722
- // Signal 1: Cluster membership small boost
723
- if (clusteredIds.has(engram.id)) {
724
- drift += 0.01;
725
- }
726
-
727
- // Signal 2: Zero edges small penalty
728
- if (edgeCount === 0) {
729
- drift -= 0.02;
730
- }
731
-
732
- // Signal 3: Long neglect → drift toward 0.3
733
- if (daysSinceAccess > CONFIDENCE_NEGLECT_DAYS && engram.confidence > 0.3) {
734
- drift -= 0.01;
735
- }
736
-
737
- // Apply with cap
738
- if (Math.abs(drift) > 0.001) {
739
- drift = Math.max(-CONFIDENCE_DRIFT_CAP, Math.min(CONFIDENCE_DRIFT_CAP, drift));
740
- const newConf = Math.max(0.15, Math.min(0.85, engram.confidence + drift));
741
- if (Math.abs(newConf - engram.confidence) > 0.001) {
742
- await this.store.updateConfidence(engram.id, newConf);
743
- result.confidenceAdjusted++;
744
- }
745
- }
746
- }
747
-
748
- // --- Phase 7: Sweep staging ---
749
- const staging = (await this.store.getEngramsByAgent(agentId, 'staging'))
750
- .filter(e => e.embedding && e.embedding.length > 0);
751
-
752
- for (const staged of staging) {
753
- const ageMs = Date.now() - staged.createdAt.getTime();
754
-
755
- // Check if this staging memory resonates with any active memory
756
- let maxSim = 0;
757
- for (const active of engrams) {
758
- if (!active.embedding || !staged.embedding) continue;
759
- const sim = cosineSimilarity(staged.embedding, active.embedding);
760
- if (sim > maxSim) maxSim = sim;
761
- }
762
-
763
- if (maxSim >= 0.6) {
764
- // Resonates — promote to active with low confidence (barely made it)
765
- await this.store.updateStage(staged.id, 'active');
766
- await this.store.updateConfidence(staged.id, 0.40);
767
- result.stagingPromoted++;
768
- } else if (ageMs > 24 * 60 * 60 * 1000) {
769
- // Over 24h and no resonance — discard
770
- await this.store.deleteEngram(staged.id);
771
- result.stagingDiscarded++;
772
- }
773
- // Otherwise: leave in staging, maybe next cycle
774
- }
775
-
776
- return result;
777
- }
778
-
779
- /**
780
- * Find clusters of semantically similar memories.
781
- * Greedy agglomerative each memory belongs to at most one cluster.
782
- * Clusters of size 2+ are returned (pairs count they link).
783
- */
784
- /**
785
- * Diameter-enforced greedy clustering.
786
- * Single-link entry (cosine SIMILARITY_THRESHOLD to any member)
787
- * + complete-link diameter (cosine MIN_PAIRWISE_COS to ALL members).
788
- * Prevents chaining where physics→biophysics→cooking = 1 cluster.
789
- */
790
- private findClusters(engrams: Engram[]): Engram[][] {
791
- const n = engrams.length;
792
- if (n < 2) return [];
793
-
794
- // Precompute pairwise cosine matrix
795
- const sim: number[][] = Array.from({ length: n }, () => Array(n).fill(0));
796
- for (let i = 0; i < n; i++) {
797
- sim[i][i] = 1;
798
- for (let j = i + 1; j < n; j++) {
799
- if (!engrams[i].embedding || !engrams[j].embedding) continue;
800
- const c = cosineSimilarity(engrams[i].embedding!, engrams[j].embedding!);
801
- sim[i][j] = c;
802
- sim[j][i] = c;
803
- }
804
- }
805
-
806
- const unassigned = new Set<number>(Array.from({ length: n }, (_, i) => i));
807
- const clusters: Engram[][] = [];
808
-
809
- const sortedIdxs = Array.from({ length: n }, (_, i) => i)
810
- .sort((a, b) => engrams[b].accessCount - engrams[a].accessCount);
811
-
812
- for (const seedIdx of sortedIdxs) {
813
- if (!unassigned.has(seedIdx)) continue;
814
- unassigned.delete(seedIdx);
815
-
816
- const clusterIdxs: number[] = [seedIdx];
817
- let added = true;
818
-
819
- while (added) {
820
- added = false;
821
- for (const candIdx of Array.from(unassigned)) {
822
- let links = false;
823
- for (const m of clusterIdxs) {
824
- if (sim[candIdx][m] >= SIMILARITY_THRESHOLD) { links = true; break; }
825
- }
826
- if (!links) continue;
827
-
828
- let passesAll = true;
829
- for (const m of clusterIdxs) {
830
- if (sim[candIdx][m] < MIN_PAIRWISE_COS) { passesAll = false; break; }
831
- }
832
- if (!passesAll) continue;
833
-
834
- clusterIdxs.push(candIdx);
835
- unassigned.delete(candIdx);
836
- added = true;
837
- }
838
- }
839
-
840
- if (clusterIdxs.length >= 2) {
841
- clusters.push(clusterIdxs.map(i => engrams[i]));
842
- } else {
843
- unassigned.add(seedIdx);
844
- }
845
- }
846
-
847
- return clusters;
848
- }
849
-
850
- /**
851
- * Compute the centroid (average embedding) of a cluster.
852
- */
853
- private computeCentroid(cluster: Engram[]): number[] {
854
- const withEmbed = cluster.filter(e => e.embedding && e.embedding.length > 0);
855
- if (withEmbed.length === 0) return [];
856
-
857
- const dim = withEmbed[0].embedding!.length;
858
- const centroid = new Array<number>(dim).fill(0);
859
- for (const e of withEmbed) {
860
- for (let i = 0; i < dim; i++) {
861
- centroid[i] += e.embedding![i];
862
- }
863
- }
864
- for (let i = 0; i < dim; i++) {
865
- centroid[i] /= withEmbed.length;
866
- }
867
- return centroid;
868
- }
869
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Sleep Cycle — offline memory consolidation.
5
+ *
6
+ * Models the brain's consolidation during sleep:
7
+ * 1. Replay — find clusters of semantically similar memories
8
+ * 2. Strengthen — reinforce edges within clusters + access-weighted boost
9
+ * 3. Bridge — create cross-cluster shortcuts between related topic areas
10
+ * 4. Decay — weaken unused edges, prune dead ones
11
+ * 5. Homeostasis — normalize outgoing edge weights to prevent hub explosion
12
+ * 6. Forget — archive/delete memories that were never retrieved (age-gated)
13
+ * 7. Sweep — promote or discard uncertain (staging) memories
14
+ *
15
+ * No artificial "summary nodes" are created. Instead, the associative
16
+ * graph gets denser where knowledge overlaps and sparser where it doesn't.
17
+ * The beam search graph walk in activation.ts naturally propagates through
18
+ * these strengthened pathways.
19
+ *
20
+ * Run between sessions or on a timer (e.g., every few hours).
21
+ */
22
+
23
+ import { cosineSimilarity } from '../core/embeddings.js';
24
+ import { strengthenAssociation, decayAssociation } from '../core/hebbian.js';
25
+ import type { Engram } from '../types/index.js';
26
+ import type { IEngramStore as EngramStore } from '../storage/store.js';
27
+ import type { ConnectionEngine } from './connections.js';
28
+ import { setConsolidationActive } from '../core/write-telemetry.js';
29
+
30
+ /** Cosine similarity for initial candidate detection (single-link entry gate) */
31
+ const SIMILARITY_THRESHOLD = 0.65;
32
+
33
+ /** Minimum pairwise cosine for cluster diameter enforcement.
34
+ * Prevents chaining: a candidate must be this similar to ALL cluster members. */
35
+ const MIN_PAIRWISE_COS = 0.50;
36
+
37
+ /** Lower threshold for cross-cluster bridge edges */
38
+ const BRIDGE_THRESHOLD = 0.25;
39
+
40
+ /** Minimum edge weight to form a new connection during replay */
41
+ const INITIAL_EDGE_WEIGHT = 0.3;
42
+
43
+ /** Boost factor for strengthening existing edges between cluster members */
44
+ const CONSOLIDATION_SIGNAL = 0.5;
45
+
46
+ /** Max new edges to create per sleep cycle (prevent graph explosion) */
47
+ const MAX_NEW_EDGES_PER_CYCLE = 50;
48
+
49
+ /** Max bridge edges per cycle (cross-cluster shortcuts) */
50
+ const MAX_BRIDGE_EDGES_PER_CYCLE = 20;
51
+
52
+ /** Edge weight below which we prune during decay */
53
+ const PRUNE_THRESHOLD = 0.01;
54
+
55
+ /** Target total outgoing edge weight per node (homeostasis) */
56
+ const HOMEOSTASIS_TARGET = 10.0;
57
+
58
+ /** Grace period before forgetting curve starts (days) */
59
+ const FORGET_GRACE_DAYS = 7;
60
+
61
+ /** Consolidation cycles before 0-access memories get archived */
62
+ const FORGET_CYCLE_THRESHOLD = 5;
63
+
64
+ /** Percentile of edge count distribution used for forgetting protection (0-1) */
65
+ const EDGE_PROTECTION_PERCENTILE = 0.25;
66
+
67
+ /** Age at which never-retrieved memories get archived (days) */
68
+ const FORGET_ARCHIVE_DAYS = 30;
69
+
70
+ /** Age at which archived, never-retrieved, unconnected memories get deleted (days) */
71
+ const FORGET_DELETE_DAYS = 90;
72
+
73
+ /** Cosine similarity above which two low-confidence memories are considered redundant.
74
+ * MiniLM-L6 paraphrases typically score 0.75-0.88 cosine; 0.85 misses most of them.
75
+ * 0.75 catches paraphrases while keeping precision above 0.60 for unrelated facts. */
76
+ const REDUNDANCY_THRESHOLD = 0.75;
77
+
78
+ /** Max redundant memories to prune per cycle (gradual, not sudden).
79
+ * Raised from 10 to 25 the eval harness runs multiple cycles anyway,
80
+ * but faster convergence reduces consolidation time for larger pools. */
81
+ const MAX_REDUNDANCY_PRUNE_PER_CYCLE = 25;
82
+
83
+ /** Max confidence drift per consolidation cycle (prevents runaway) */
84
+ const CONFIDENCE_DRIFT_CAP = 0.03;
85
+
86
+ /**
87
+ * Content fade Paper 1 (PLOS Comp Biology, storage degradation).
88
+ * Memories whose content has not been recalled in a long time get coarsened:
89
+ * the content string is trimmed to FADE_KEEP_CHARS with a "[faded]" marker,
90
+ * while concept, tags, and embedding are preserved.
91
+ *
92
+ * Rationale: human memory degrades the detailed surface but retains the gist
93
+ * and the cue-association pathway. The faded engram still surfaces by tag/concept
94
+ * BM25 and by vector similarity (embedding preserved), but the full content body
95
+ * is gone which is the correct behavior for a memory that has not been useful
96
+ * enough to rehearse in a long time.
97
+ *
98
+ * Fade comes BEFORE archive in the lifecycle (active fading archived deleted),
99
+ * applied only to engrams that have been accessed at least once. Engrams that have
100
+ * never been accessed go straight to archive via the standard forgetting path —
101
+ * they have no surface worth preserving.
102
+ *
103
+ * Env: AWM_FADE_DAYS_SINCE_ACCESS (default 45),
104
+ * AWM_FADE_KEEP_CHARS (default 150),
105
+ * AWM_FADE_MIN_CONTENT_LEN (default 250),
106
+ * AWM_FADE_MAX_PER_CYCLE (default 25).
107
+ */
108
+ const FADE_DAYS_SINCE_ACCESS = Number(process.env.AWM_FADE_DAYS_SINCE_ACCESS ?? 45);
109
+ const FADE_KEEP_CHARS = Number(process.env.AWM_FADE_KEEP_CHARS ?? 150);
110
+ const FADE_MIN_CONTENT_LEN = Number(process.env.AWM_FADE_MIN_CONTENT_LEN ?? 250);
111
+ const FADE_MAX_PER_CYCLE = Number(process.env.AWM_FADE_MAX_PER_CYCLE ?? 25);
112
+ const FADE_PROTECTED_ACCESS_COUNT = 10;
113
+ const FADE_MARKER = '… [faded]';
114
+ const FADE_PROTECTED_CLASSES = new Set(['canonical', 'structural']);
115
+
116
+ /** Days without recall before confidence starts drifting down */
117
+ const CONFIDENCE_NEGLECT_DAYS = 30;
118
+
119
+ export interface ConsolidationResult {
120
+ clustersFound: number;
121
+ edgesStrengthened: number;
122
+ edgesCreated: number;
123
+ bridgesCreated: number;
124
+ edgesDecayed: number;
125
+ edgesPruned: number;
126
+ edgesNormalized: number;
127
+ memoriesForgotten: number;
128
+ memoriesArchived: number;
129
+ memoriesFaded: number;
130
+ redundancyPruned: number;
131
+ confidenceAdjusted: number;
132
+ stagingPromoted: number;
133
+ stagingDiscarded: number;
134
+ engramsProcessed: number;
135
+ synthesesCreated: number;
136
+ }
137
+
138
+ const MAX_SYNTHESES_PER_CYCLE = 5;
139
+ const MIN_CLUSTER_SIZE_FOR_SYNTHESIS = 3;
140
+
141
+ /** Shared stopwords for synthesis keyword extraction */
142
+ const SYNTH_STOPWORDS = new Set(['the', 'is', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'for',
143
+ 'on', 'with', 'that', 'this', 'it', 'was', 'are', 'be', 'has', 'had', 'but', 'not', 'from',
144
+ 'by', 'as', 'at', 'i', 'you', 'we', 'my', 'your', 'can', 'will', 'do', 'did', 'if', 'user',
145
+ 'assistant', 'would', 'like', 'just', 'also', 'about', 'really', 'think', 'know', 'want',
146
+ 'here', 'there', 'some', 'more', 'very', 'been', 'have', 'what', 'when', 'how', 'they']);
147
+
148
+ export class ConsolidationEngine {
149
+ private store: EngramStore;
150
+ private connectionEngine: ConnectionEngine | null;
151
+
152
+ /**
153
+ * Optional `connectionEngine` is drained at the start of each
154
+ * consolidation pass (v0.8.2). Connection discovery is no longer
155
+ * triggered per write the work batches into the existing sleep cycle
156
+ * so writes return in ~50 ms instead of being pegged by an inline
157
+ * embed + rerank cycle (~200-500 ms/write under load).
158
+ */
159
+ constructor(store: EngramStore, connectionEngine?: ConnectionEngine) {
160
+ this.store = store;
161
+ this.connectionEngine = connectionEngine ?? null;
162
+ }
163
+
164
+ /**
165
+ * Run a full sleep cycle for an agent.
166
+ *
167
+ * Phase 0: Connection drain discover associations for engrams enqueued
168
+ * by writes since the last consolidation (v0.8.2)
169
+ * Phase 1: Replayfind clusters of semantically similar memories
170
+ * Phase 2: Strengthenreinforce edges within clusters (access-weighted)
171
+ * Phase 3: Bridgecreate cross-cluster shortcuts
172
+ * Phase 4: Decayweaken unused edges, prune dead ones
173
+ * Phase 5: Homeostasisnormalize outgoing edge weights per node
174
+ * Phase 5.5: Content fade coarsen un-recalled engrams (Paper 1)
175
+ * Phase 6: Forget — archive/delete memories never retrieved (age-gated)
176
+ * Phase 6.5: Redundancy prune
177
+ * Phase 6.7: Confidence drift adjust confidence based on structural signals
178
+ * Phase 7: Sweep — check staging buffer for resonance
179
+ */
180
+ async consolidate(agentId: string): Promise<ConsolidationResult> {
181
+ // D1 telemetry (2026-07-30): flag the cycle so slow-write reports can
182
+ // attribute stalls to in-process consolidation. try/finally guarantees
183
+ // the flag clears even when a phase throws.
184
+ setConsolidationActive(true, agentId);
185
+ const tCycleStart = performance.now();
186
+ try {
187
+ return await this.consolidateInner(agentId);
188
+ } finally {
189
+ setConsolidationActive(false);
190
+ const ms = Math.round(performance.now() - tCycleStart);
191
+ if (ms > 5000) {
192
+ process.stderr.write(`[awm] consolidation cycle for ${agentId} took ${ms}ms\n`);
193
+ }
194
+ }
195
+ }
196
+
197
+ private async consolidateInner(agentId: string): Promise<ConsolidationResult> {
198
+ // --- Phase 0: Connection drain ---
199
+ // Discover associations for engrams enqueued by writes since the last
200
+ // consolidation. Per-write inline discovery was removed in v0.8.2
201
+ // because each findConnections() call runs a full activation cycle
202
+ // (embed + rerank, ~200-500 ms event-loop block per write).
203
+ if (this.connectionEngine) {
204
+ try { await this.connectionEngine.processQueue(); } catch { /* best-effort */ }
205
+ }
206
+ const result: ConsolidationResult = {
207
+ clustersFound: 0,
208
+ edgesStrengthened: 0,
209
+ edgesCreated: 0,
210
+ bridgesCreated: 0,
211
+ edgesDecayed: 0,
212
+ edgesPruned: 0,
213
+ edgesNormalized: 0,
214
+ memoriesForgotten: 0,
215
+ memoriesArchived: 0,
216
+ memoriesFaded: 0,
217
+ redundancyPruned: 0,
218
+ confidenceAdjusted: 0,
219
+ stagingPromoted: 0,
220
+ stagingDiscarded: 0,
221
+ engramsProcessed: 0,
222
+ synthesesCreated: 0,
223
+ };
224
+
225
+ // --- Phase 1: Replay ---
226
+ // Get all active engrams, backfill missing embeddings
227
+ const allActive = await this.store.getEngramsByAgent(agentId, 'active');
228
+ const needsEmbedding = allActive.filter(e => !e.embedding || e.embedding.length === 0);
229
+ if (needsEmbedding.length > 0) {
230
+ try {
231
+ const { embedBatch, getModelId } = await import('../core/embeddings.js');
232
+ const BATCH_SIZE = 32;
233
+ const modelId = getModelId?.() ?? 'unknown';
234
+ for (let b = 0; b < needsEmbedding.length; b += BATCH_SIZE) {
235
+ const batch = needsEmbedding.slice(b, b + BATCH_SIZE);
236
+ try {
237
+ const texts = batch.map(e => `${e.concept} ${e.content}`);
238
+ const vecs = await embedBatch(texts);
239
+ for (let j = 0; j < batch.length; j++) {
240
+ await this.store.updateEmbedding(batch[j].id, vecs[j], modelId);
241
+ batch[j].embedding = vecs[j];
242
+ }
243
+ } catch { /* batch failed, skip non-fatal */ }
244
+ }
245
+ if (needsEmbedding.length > 0) {
246
+ console.log(`[consolidation] Backfilled ${needsEmbedding.filter(e => e.embedding?.length).length}/${needsEmbedding.length} embeddings (model: ${modelId})`);
247
+ }
248
+ } catch { /* embeddings module unavailable */ }
249
+ }
250
+ const engrams = allActive.filter(e => e.embedding && e.embedding.length > 0);
251
+
252
+ result.engramsProcessed = engrams.length;
253
+ // Clustering requires at least two engrams to form pairs. Per-engram phases
254
+ // (fade, forget, confidence drift, staging sweep) still run for singletons —
255
+ // they don't need a graph.
256
+ const clusters = engrams.length >= 2 ? this.findClusters(engrams) : [];
257
+ result.clustersFound = clusters.length;
258
+
259
+ // --- Phase 2: Strengthen (access-weighted) ---
260
+ // Memories that are retrieved more often get stronger consolidation.
261
+ // This mirrors how the brain preferentially consolidates practiced memories.
262
+ let newEdges = 0;
263
+ for (const cluster of clusters) {
264
+ for (let i = 0; i < cluster.length; i++) {
265
+ for (let j = i + 1; j < cluster.length; j++) {
266
+ const a = cluster[i];
267
+ const b = cluster[j];
268
+
269
+ // Access-weighted signal: more retrieved = stronger consolidation
270
+ const accessFactor = Math.min(
271
+ 1.0,
272
+ 0.3 + 0.7 * Math.log1p(a.accessCount + b.accessCount) / Math.log1p(20),
273
+ );
274
+
275
+ const existing = await this.store.getAssociation(a.id, b.id);
276
+ if (existing) {
277
+ const newWeight = strengthenAssociation(
278
+ existing.weight, CONSOLIDATION_SIGNAL * accessFactor, 0.25,
279
+ );
280
+ await this.store.upsertAssociation(
281
+ a.id, b.id, newWeight, existing.type, existing.confidence,
282
+ );
283
+ result.edgesStrengthened++;
284
+ } else if (newEdges < MAX_NEW_EDGES_PER_CYCLE) {
285
+ await this.store.upsertAssociation(
286
+ a.id, b.id, INITIAL_EDGE_WEIGHT * accessFactor, 'connection',
287
+ );
288
+ newEdges++;
289
+ result.edgesCreated++;
290
+ }
291
+ }
292
+ }
293
+ }
294
+
295
+ // --- Phase 2.5: Two types of synthesis ---
296
+ //
297
+ // Type A: SESSION SYNTHESIS (perfect recall)
298
+ // Groups by shared metadata tags (sid=, proj=, topic=).
299
+ // Summarizes what happened in a conversation/project session.
300
+ // Helps find specific facts by providing a topical anchor.
301
+ //
302
+ // Type B: PATTERN SYNTHESIS (novel recall)
303
+ // Uses the existing vector-similarity clusters.
304
+ // Finds structural patterns across disparate topics.
305
+ // "Debugging X by Y" + "Resolving A by B" → pattern: "conflict → ordering"
306
+ // Lower confidence these are speculative connections, not facts.
307
+
308
+ let synthCount = 0;
309
+
310
+ // --- Type A: Session synthesis (tag-based grouping) ---
311
+ // Group engrams by shared session/project tags, NOT vector similarity
312
+ const tagGroups = new Map<string, Engram[]>();
313
+ for (const e of engrams) {
314
+ if (e.tags.includes('synth=true')) continue; // Skip existing syntheses
315
+ for (const tag of e.tags) {
316
+ if (tag.startsWith('sid=') || tag.startsWith('proj=') || tag.startsWith('topic=')) {
317
+ const group = tagGroups.get(tag) ?? [];
318
+ group.push(e);
319
+ tagGroups.set(tag, group);
320
+ }
321
+ }
322
+ }
323
+
324
+ for (const [tag, group] of tagGroups) {
325
+ if (group.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
326
+ if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
327
+
328
+ // Check if a synthesis for this tag already exists
329
+ const existing = engrams.find(e =>
330
+ e.tags.includes('synth=true') && e.tags.includes(tag)
331
+ );
332
+ if (existing) continue;
333
+
334
+ // Extract key terms from this group
335
+ const wordCounts = new Map<string, number>();
336
+ for (const e of group) {
337
+ const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
338
+ for (const w of words) {
339
+ if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
340
+ wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
341
+ }
342
+ }
343
+ }
344
+ const keyTerms = [...wordCounts.entries()]
345
+ .sort((a, b) => b[1] - a[1])
346
+ .slice(0, 15)
347
+ .map(([word]) => word);
348
+
349
+ // Extract unique concepts (deduplicated)
350
+ const concepts = [...new Set(group.map(e => e.concept))];
351
+
352
+ const synthContent = [
353
+ `Session summary (${tag}, ${group.length} turns).`,
354
+ `Key topics: ${keyTerms.slice(0, 8).join(', ')}.`,
355
+ `Discussed: ${keyTerms.slice(8).join(', ')}.`,
356
+ ].join(' ');
357
+
358
+ const synthEngram = await this.store.createEngram({
359
+ agentId,
360
+ concept: `session: ${tag} (${keyTerms.slice(0, 3).join(', ')})`,
361
+ content: synthContent,
362
+ tags: [tag, 'synth=true', 'synth-type=session'],
363
+ salience: 0.6,
364
+ confidence: 0.55,
365
+ memoryType: 'semantic',
366
+ });
367
+
368
+ // Set embedding to group centroid
369
+ const centroid = this.computeCentroid(group);
370
+ if (centroid.length > 0) {
371
+ await this.store.updateEmbedding(synthEngram.id, centroid);
372
+ }
373
+
374
+ // Link to sources
375
+ for (const source of group.slice(0, 10)) { // Cap links to prevent explosion
376
+ await this.store.upsertAssociation(synthEngram.id, source.id, 0.4, 'causal');
377
+ }
378
+
379
+ synthCount++;
380
+ result.synthesesCreated++;
381
+ }
382
+
383
+ // --- Type B: Pattern synthesis (vector-similarity clusters, speculative) ---
384
+ // Only create these for clusters where members come from DIFFERENT sessions/projects.
385
+ // This finds cross-domain patterns: "debugging technique A" + "architecture pattern B"
386
+ for (const cluster of clusters) {
387
+ if (cluster.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
388
+ if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
389
+ if (cluster.some(e => e.tags.includes('synth=true'))) continue;
390
+
391
+ // Only create pattern synthesis if cluster spans multiple sessions
392
+ const sessionTags = new Set<string>();
393
+ for (const e of cluster) {
394
+ for (const tag of e.tags) {
395
+ if (tag.startsWith('sid=') || tag.startsWith('proj=')) sessionTags.add(tag);
396
+ }
397
+ }
398
+ if (sessionTags.size < 2) continue; // Same session → skip (Type A handles it)
399
+
400
+ const wordCounts = new Map<string, number>();
401
+ for (const e of cluster) {
402
+ const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
403
+ for (const w of words) {
404
+ if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
405
+ wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
406
+ }
407
+ }
408
+ }
409
+ const keyTerms = [...wordCounts.entries()]
410
+ .sort((a, b) => b[1] - a[1])
411
+ .slice(0, 10)
412
+ .map(([word]) => word);
413
+
414
+ const synthContent = [
415
+ `Pattern across ${sessionTags.size} sessions (${cluster.length} memories).`,
416
+ `Common themes: ${keyTerms.join(', ')}.`,
417
+ ].join(' ');
418
+
419
+ const synthEngram = await this.store.createEngram({
420
+ agentId,
421
+ concept: `pattern: ${keyTerms.slice(0, 3).join(', ')}`,
422
+ content: synthContent,
423
+ tags: [...sessionTags, 'synth=true', 'synth-type=pattern'],
424
+ salience: 0.5, // Lower speculative
425
+ confidence: 0.4, // Lower these are hypotheses not facts
426
+ memoryType: 'semantic',
427
+ });
428
+
429
+ const centroid = this.computeCentroid(cluster);
430
+ if (centroid.length > 0) {
431
+ await this.store.updateEmbedding(synthEngram.id, centroid);
432
+ }
433
+
434
+ for (const source of cluster.slice(0, 8)) {
435
+ await this.store.upsertAssociation(synthEngram.id, source.id, 0.3, 'bridge');
436
+ }
437
+
438
+ synthCount++;
439
+ result.synthesesCreated++;
440
+ }
441
+
442
+ // --- Phase 3: Direct cross-cluster bridging ---
443
+ // Find the closest pair of memories between each cluster pair and bridge them.
444
+ if (clusters.length >= 2) {
445
+ const MIN_BRIDGE_SIM = 0.15;
446
+ let bridges = 0;
447
+ for (let i = 0; i < clusters.length && bridges < MAX_BRIDGE_EDGES_PER_CYCLE; i++) {
448
+ for (let j = i + 1; j < clusters.length && bridges < MAX_BRIDGE_EDGES_PER_CYCLE; j++) {
449
+ let bestSim = -1;
450
+ let bestA: Engram | null = null;
451
+ let bestB: Engram | null = null;
452
+ for (const a of clusters[i]) {
453
+ if (!a.embedding) continue;
454
+ for (const b of clusters[j]) {
455
+ if (!b.embedding) continue;
456
+ const s = cosineSimilarity(a.embedding, b.embedding);
457
+ if (s > bestSim) { bestSim = s; bestA = a; bestB = b; }
458
+ }
459
+ }
460
+ if (bestA && bestB && bestSim > MIN_BRIDGE_SIM) {
461
+ const existing = await this.store.getAssociation(bestA.id, bestB.id);
462
+ if (!existing) {
463
+ await this.store.upsertAssociation(bestA.id, bestB.id, bestSim, 'bridge');
464
+ await this.store.upsertAssociation(bestB.id, bestA.id, bestSim, 'bridge');
465
+ bridges++;
466
+ result.bridgesCreated++;
467
+ }
468
+ }
469
+ }
470
+ }
471
+ }
472
+
473
+ // --- Phase 4: Decay (confidence-modulated) ---
474
+ // High-confidence edges decay slower. This means edges between memories
475
+ // that received positive feedback are more durable just like how
476
+ // practiced memories are more resistant to forgetting in the brain.
477
+ // Base half-life: 7 days. High-confidence (0.8+) gets up to 30 days.
478
+ const engramMap = new Map(engrams.map(e => [e.id, e]));
479
+ const associations = await this.store.getAllAssociations(agentId);
480
+ for (const assoc of associations) {
481
+ const daysSince =
482
+ (Date.now() - assoc.lastActivated.getTime()) / (1000 * 60 * 60 * 24);
483
+ if (daysSince < 0.5) continue;
484
+
485
+ const fromEngram = engramMap.get(assoc.fromEngramId);
486
+ const toEngram = engramMap.get(assoc.toEngramId);
487
+ const fromConf = fromEngram?.confidence ?? 0.5;
488
+ const toConf = toEngram?.confidence ?? 0.5;
489
+ const maxConf = Math.max(fromConf, toConf);
490
+ const maxAccess = Math.max(fromEngram?.accessCount ?? 0, toEngram?.accessCount ?? 0);
491
+ const accessBoost = Math.min(2.0, 1.0 + 0.5 * Math.log1p(maxAccess));
492
+ const halfLifeDays = Math.min(
493
+ 7 * (1 + 2 * Math.max(0, (maxConf - 0.5) / 0.5)) * accessBoost,
494
+ 42 // Hard cap: 6 weeks max
495
+ );
496
+
497
+ const newWeight = decayAssociation(assoc.weight, daysSince, halfLifeDays);
498
+ if (newWeight < PRUNE_THRESHOLD) {
499
+ await this.store.deleteAssociation(assoc.id);
500
+ result.edgesPruned++;
501
+ } else if (Math.abs(newWeight - assoc.weight) > 0.001) {
502
+ await this.store.upsertAssociation(
503
+ assoc.fromEngramId, assoc.toEngramId,
504
+ newWeight, assoc.type, assoc.confidence,
505
+ );
506
+ result.edgesDecayed++;
507
+ }
508
+ }
509
+
510
+ // --- Phase 5: Synaptic homeostasis ---
511
+ // Normalize total outgoing edge weight per node to prevent hub explosion.
512
+ // Nodes with many strong edges get scaled down so relative weights stay meaningful.
513
+ const engramIds = new Set(engrams.map(e => e.id));
514
+ for (const id of engramIds) {
515
+ const outgoing = await this.store.getOutgoingAssociations(id);
516
+ const totalWeight = outgoing.reduce((sum, a) => sum + a.weight, 0);
517
+ if (totalWeight > HOMEOSTASIS_TARGET) {
518
+ const scale = HOMEOSTASIS_TARGET / totalWeight;
519
+ for (const edge of outgoing) {
520
+ const newWeight = edge.weight * scale;
521
+ if (newWeight < PRUNE_THRESHOLD) {
522
+ await this.store.deleteAssociation(edge.id);
523
+ result.edgesPruned++;
524
+ } else {
525
+ await this.store.upsertAssociation(
526
+ edge.fromEngramId, edge.toEngramId,
527
+ newWeight, edge.type, edge.confidence,
528
+ );
529
+ }
530
+ }
531
+ result.edgesNormalized++;
532
+ }
533
+ }
534
+
535
+ // --- Phase 5.5: Content fade (Paper 1: storage degradation) ---
536
+ // Coarsens engrams that have been accessed before but have gone stale.
537
+ // Trims content surface area while preserving concept, tags, and embedding so
538
+ // the engram still participates in BM25 + vector recall — just with less
539
+ // body to score against. This models how human memory loses surface detail
540
+ // while retaining cue-association pathways.
541
+ //
542
+ // Fade applies only to accessed engrams (the never-accessed ones go straight
543
+ // to archive via Phase 6). Heavily-used (accessCount >= FADE_PROTECTED_ACCESS_COUNT),
544
+ // canonical, structural, and short-content engrams are excluded.
545
+ let fadeCount = 0;
546
+ for (const engram of engrams) {
547
+ if (fadeCount >= FADE_MAX_PER_CYCLE) break;
548
+ if (engram.accessCount === 0) continue;
549
+ if (engram.accessCount >= FADE_PROTECTED_ACCESS_COUNT) continue;
550
+ if (FADE_PROTECTED_CLASSES.has(engram.memoryClass)) continue;
551
+ if (engram.retracted) continue;
552
+ if (engram.content.length < FADE_MIN_CONTENT_LEN) continue;
553
+ if (engram.content.endsWith(FADE_MARKER)) continue; // Already faded
554
+ const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
555
+ if (daysSinceAccess < FADE_DAYS_SINCE_ACCESS) continue;
556
+
557
+ const trimmed = engram.content.slice(0, FADE_KEEP_CHARS).trimEnd() + FADE_MARKER;
558
+ await this.store.updateContent(engram.id, trimmed);
559
+ await this.store.updateStage(engram.id, 'fading');
560
+ engram.content = trimmed;
561
+ engram.stage = 'fading';
562
+ result.memoriesFaded++;
563
+ fadeCount++;
564
+ }
565
+
566
+ // --- Phase 6: Forgetting (age-gated) ---
567
+ // Models how human memory actually works:
568
+ // - New memories get a grace period (too new to judge)
569
+ // - Retrieval acts as rehearsal — resets the forgetting clock
570
+ // - Well-connected memories persist (edges = integration into knowledge)
571
+ // - Old, isolated, unretrieved memories fade to archive (not deleted)
572
+ // - Archived memories can still be recovered via deep search
573
+ // - Only truly orphaned, ancient memories get deleted
574
+ //
575
+ // Key insight: outdated memories still have value as historical context.
576
+ // "We used to use X" helps explain why we now use Y.
577
+ // Compute edge count percentile for relative protection threshold.
578
+ // With avg 12 edges/node, an absolute threshold of 3 protects everything.
579
+ // Use 25th percentile so "weakly connected" is relative to actual graph density.
580
+ const edgeCounts = await Promise.all(engrams.map(async e => await this.store.countAssociationsFor(e.id)));
581
+ edgeCounts.sort((a, b) => a - b);
582
+ const percentileIdx = Math.floor(edgeCounts.length * EDGE_PROTECTION_PERCENTILE);
583
+ const baseEdgeThreshold = edgeCounts.length > 0 ? edgeCounts[percentileIdx] : 3;
584
+
585
+ // Get consolidation cycle count for cycle-based archiving
586
+ const cycleCount = await this.store.getConsolidationCycleCount(agentId);
587
+
588
+ for (const engram of engrams) {
589
+ // Skip engrams that just transitioned to 'fading' in Phase 5.5 this cycle.
590
+ // Fade is intended as an intermediate stage between active and archived —
591
+ // the engram should live in 'fading' for at least one more recall window
592
+ // before being eligible for archival. Without this guard, a same-cycle
593
+ // fade-then-archive sequence collapses the active pool to zero (observed
594
+ // in the stress test scale phase around cycle 90).
595
+ if (engram.stage !== 'active') continue;
596
+
597
+ const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
598
+ if (ageDays < FORGET_GRACE_DAYS) continue; // Grace period — too new to judge
599
+
600
+ const edgeCount = await this.store.countAssociationsFor(engram.id);
601
+
602
+ // Use relative threshold (percentile-based) instead of absolute.
603
+ // High-confidence memories need fewer edges to survive.
604
+ const confReduction = engram.confidence > 0.7
605
+ ? Math.min(0.6, (engram.confidence - 0.7) * 2)
606
+ : 0;
607
+ const edgeProtectionThreshold = Math.max(1, Math.round(baseEdgeThreshold * (1 - confReduction)));
608
+ if (edgeCount > edgeProtectionThreshold) continue;
609
+
610
+ // Cycle-based archive: 0-access memories archived after N cycles
611
+ // regardless of age. Handles small pools where time thresholds are too generous.
612
+ if (engram.accessCount === 0 && cycleCount >= FORGET_CYCLE_THRESHOLD) {
613
+ await this.store.updateStage(engram.id, 'archived');
614
+ result.memoriesArchived++;
615
+ continue;
616
+ }
617
+
618
+ // Compute effective forgetting threshold based on memory strength signals.
619
+ // Rehearsal (access + feedback) extends protection but NEVER makes immortal.
620
+ // Models a sharp 20-year senior dev: confirmed knowledge persists for months/years.
621
+ // - Base: FORGET_ARCHIVE_DAYS (30 days)
622
+ // - Access extends by log-scaled factor: 5 accesses ≈ 2x, 10 ≈ 2.5x
623
+ // - Confidence modulates up to 4x (0.5→1x, 0.7→2.6x, 0.8→3.4x, 1.0→4x)
624
+ // - Hard cap: 12x base (360 days) — even the sharpest memory fades after a year
625
+ const accessFactor = 1 + Math.log1p(engram.accessCount) * 0.6;
626
+ const confFactor = 1 + 3 * Math.max(0, (engram.confidence - 0.5) / 0.5);
627
+ const effectiveArchiveDays = Math.min(
628
+ FORGET_ARCHIVE_DAYS * accessFactor * confFactor,
629
+ FORGET_ARCHIVE_DAYS * 12, // Hard cap: 12x base (360 days)
630
+ );
631
+
632
+ const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
633
+
634
+ if (engram.accessCount === 0 && ageDays > FORGET_ARCHIVE_DAYS) {
635
+ // Never retrieved, old, weakly connected → archive
636
+ await this.store.updateStage(engram.id, 'archived');
637
+ result.memoriesArchived++;
638
+ } else if (engram.accessCount > 0 && daysSinceAccess > effectiveArchiveDays) {
639
+ // Accessed before but not recently enough given its strength — archive
640
+ await this.store.updateStage(engram.id, 'archived');
641
+ result.memoriesArchived++;
642
+ }
643
+ }
644
+
645
+ // Check archived memories for deletion only truly orphaned ancient ones
646
+ const archived = await this.store.getEngramsByAgent(agentId, 'archived');
647
+ for (const engram of archived) {
648
+ const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
649
+ const edgeCount = await this.store.countAssociationsFor(engram.id);
650
+
651
+ if (engram.accessCount === 0 && ageDays > FORGET_DELETE_DAYS && edgeCount === 0) {
652
+ // Very old, never accessed, completely isolated → truly forgotten
653
+ await this.store.deleteEngram(engram.id);
654
+ result.memoriesForgotten++;
655
+ }
656
+ // Otherwise: stay archived — still searchable, just not in active recall
657
+ }
658
+
659
+ // --- Phase 6.5: Redundancy pruning ---
660
+ // A senior dev doesn't store 30 nearly-identical memories. When multiple
661
+ // low-confidence memories are semantically redundant (cosine > 0.85), keep
662
+ // only the one with highest accessCount + confidence and archive the rest.
663
+ // This naturally defeats volume-based attacks (narcissistic interference,
664
+ // spam) while improving signal-to-noise ratio for linked memories.
665
+ // High-confidence memories (feedback-confirmed) are never pruned — they
666
+ // represent verified knowledge worth keeping even if similar.
667
+ // Only consider memories that are both low-confidence AND rarely accessed.
668
+ // Memories retrieved 3+ times have proven useful — they stay even if similar
669
+ // to others. This prevents pruning seed memories that match bulk templates.
670
+ const lowConfEngrams = engrams.filter(e =>
671
+ e.confidence < 0.6 && e.accessCount < 3 && e.embedding && e.embedding.length > 0);
672
+ const pruned = new Set<string>();
673
+ let redundancyCount = 0;
674
+
675
+ // Sort by quality: highest accessCount + confidence first (survivors)
676
+ const sortedLow = [...lowConfEngrams].sort((a, b) =>
677
+ (b.accessCount + b.confidence * 10) - (a.accessCount + a.confidence * 10));
678
+
679
+ for (let i = 0; i < sortedLow.length && redundancyCount < MAX_REDUNDANCY_PRUNE_PER_CYCLE; i++) {
680
+ if (pruned.has(sortedLow[i].id)) continue;
681
+ for (let j = i + 1; j < sortedLow.length && redundancyCount < MAX_REDUNDANCY_PRUNE_PER_CYCLE; j++) {
682
+ if (pruned.has(sortedLow[j].id)) continue;
683
+ if (!sortedLow[i].embedding || !sortedLow[j].embedding) continue;
684
+
685
+ const sim = cosineSimilarity(sortedLow[i].embedding!, sortedLow[j].embedding!);
686
+ if (sim >= REDUNDANCY_THRESHOLD) {
687
+ const survivorId = sortedLow[i].id;
688
+ const prunedId = sortedLow[j].id;
689
+
690
+ // Transfer associations from pruned memory to survivor
691
+ const prunedEdges = await this.store.getAssociationsFor(prunedId);
692
+ for (const edge of prunedEdges) {
693
+ const peerId = edge.fromEngramId === prunedId ? edge.toEngramId : edge.fromEngramId;
694
+ if (peerId === survivorId) continue; // Skip self-loops
695
+ await this.store.upsertAssociation(survivorId, peerId, edge.weight, edge.type, edge.confidence);
696
+ }
697
+
698
+ // Merge tags from pruned to survivor
699
+ const survivor = sortedLow[i];
700
+ const prunedMem = sortedLow[j];
701
+ const mergedTags = [...new Set([...survivor.tags, ...prunedMem.tags])];
702
+ if (mergedTags.length > survivor.tags.length) {
703
+ await this.store.updateTags(survivorId, mergedTags);
704
+ }
705
+
706
+ // Archive the lower-quality duplicate
707
+ await this.store.updateStage(prunedId, 'archived');
708
+ pruned.add(prunedId);
709
+ redundancyCount++;
710
+ }
711
+ }
712
+ }
713
+ result.redundancyPruned = redundancyCount;
714
+
715
+ // --- Phase 6.7: Confidence drift ---
716
+ // Adjust confidence based on structural signals that emerge from the graph.
717
+ // This makes confidence evolve over time without explicit feedback calls.
718
+ //
719
+ // Three signals:
720
+ // 1. Well-clustered memories (appeared in 1+ clusters) get a small boost
721
+ // — they're integrated into the knowledge graph, likely valuable.
722
+ // 2. Isolated memories (0 edges after consolidation) get a small penalty
723
+ // nothing connects to them, possibly noise.
724
+ // 3. Neglected memories (not recalled in 30+ days) drift toward 0.3
725
+ // — if the system never needs them, they're probably not important.
726
+ //
727
+ // All adjustments are capped at ±0.03 per cycle to prevent runaway.
728
+ // Confidence is floored at 0.15 (never reaches 0 — retraction handles that).
729
+ // Confidence is capped at 0.85 (only explicit feedback can push above).
730
+ const clusteredIds = new Set<string>();
731
+ for (const cluster of clusters) {
732
+ for (const e of cluster) clusteredIds.add(e.id);
733
+ }
734
+
735
+ for (const engram of engrams) {
736
+ let drift = 0;
737
+ const edgeCount = await this.store.countAssociationsFor(engram.id);
738
+ const daysSinceAccess = (Date.now() - engram.lastAccessed.getTime()) / (1000 * 60 * 60 * 24);
739
+
740
+ // Signal 1: Cluster membership small boost
741
+ if (clusteredIds.has(engram.id)) {
742
+ drift += 0.01;
743
+ }
744
+
745
+ // Signal 2: Zero edges → small penalty
746
+ if (edgeCount === 0) {
747
+ drift -= 0.02;
748
+ }
749
+
750
+ // Signal 3: Long neglect drift toward 0.3
751
+ if (daysSinceAccess > CONFIDENCE_NEGLECT_DAYS && engram.confidence > 0.3) {
752
+ drift -= 0.01;
753
+ }
754
+
755
+ // Apply with cap
756
+ if (Math.abs(drift) > 0.001) {
757
+ drift = Math.max(-CONFIDENCE_DRIFT_CAP, Math.min(CONFIDENCE_DRIFT_CAP, drift));
758
+ const newConf = Math.max(0.15, Math.min(0.85, engram.confidence + drift));
759
+ if (Math.abs(newConf - engram.confidence) > 0.001) {
760
+ await this.store.updateConfidence(engram.id, newConf);
761
+ result.confidenceAdjusted++;
762
+ }
763
+ }
764
+ }
765
+
766
+ // --- Phase 7: Sweep staging ---
767
+ const staging = (await this.store.getEngramsByAgent(agentId, 'staging'))
768
+ .filter(e => e.embedding && e.embedding.length > 0);
769
+
770
+ for (const staged of staging) {
771
+ const ageMs = Date.now() - staged.createdAt.getTime();
772
+
773
+ // Check if this staging memory resonates with any active memory
774
+ let maxSim = 0;
775
+ for (const active of engrams) {
776
+ if (!active.embedding || !staged.embedding) continue;
777
+ const sim = cosineSimilarity(staged.embedding, active.embedding);
778
+ if (sim > maxSim) maxSim = sim;
779
+ }
780
+
781
+ if (maxSim >= 0.6) {
782
+ // Resonates promote to active with low confidence (barely made it)
783
+ await this.store.updateStage(staged.id, 'active');
784
+ await this.store.updateConfidence(staged.id, 0.40);
785
+ result.stagingPromoted++;
786
+ } else if (ageMs > 24 * 60 * 60 * 1000) {
787
+ // Over 24h and no resonance discard
788
+ await this.store.deleteEngram(staged.id);
789
+ result.stagingDiscarded++;
790
+ }
791
+ // Otherwise: leave in staging, maybe next cycle
792
+ }
793
+
794
+ return result;
795
+ }
796
+
797
+ /**
798
+ * Find clusters of semantically similar memories.
799
+ * Greedy agglomerative — each memory belongs to at most one cluster.
800
+ * Clusters of size 2+ are returned (pairs count — they link).
801
+ */
802
+ /**
803
+ * Diameter-enforced greedy clustering.
804
+ * Single-link entry (cosine ≥ SIMILARITY_THRESHOLD to any member)
805
+ * + complete-link diameter (cosine ≥ MIN_PAIRWISE_COS to ALL members).
806
+ * Prevents chaining where physics→biophysics→cooking = 1 cluster.
807
+ */
808
+ private findClusters(engrams: Engram[]): Engram[][] {
809
+ const n = engrams.length;
810
+ if (n < 2) return [];
811
+
812
+ // Precompute pairwise cosine matrix
813
+ const sim: number[][] = Array.from({ length: n }, () => Array(n).fill(0));
814
+ for (let i = 0; i < n; i++) {
815
+ sim[i][i] = 1;
816
+ for (let j = i + 1; j < n; j++) {
817
+ if (!engrams[i].embedding || !engrams[j].embedding) continue;
818
+ const c = cosineSimilarity(engrams[i].embedding!, engrams[j].embedding!);
819
+ sim[i][j] = c;
820
+ sim[j][i] = c;
821
+ }
822
+ }
823
+
824
+ const unassigned = new Set<number>(Array.from({ length: n }, (_, i) => i));
825
+ const clusters: Engram[][] = [];
826
+
827
+ const sortedIdxs = Array.from({ length: n }, (_, i) => i)
828
+ .sort((a, b) => engrams[b].accessCount - engrams[a].accessCount);
829
+
830
+ for (const seedIdx of sortedIdxs) {
831
+ if (!unassigned.has(seedIdx)) continue;
832
+ unassigned.delete(seedIdx);
833
+
834
+ const clusterIdxs: number[] = [seedIdx];
835
+ let added = true;
836
+
837
+ while (added) {
838
+ added = false;
839
+ for (const candIdx of Array.from(unassigned)) {
840
+ let links = false;
841
+ for (const m of clusterIdxs) {
842
+ if (sim[candIdx][m] >= SIMILARITY_THRESHOLD) { links = true; break; }
843
+ }
844
+ if (!links) continue;
845
+
846
+ let passesAll = true;
847
+ for (const m of clusterIdxs) {
848
+ if (sim[candIdx][m] < MIN_PAIRWISE_COS) { passesAll = false; break; }
849
+ }
850
+ if (!passesAll) continue;
851
+
852
+ clusterIdxs.push(candIdx);
853
+ unassigned.delete(candIdx);
854
+ added = true;
855
+ }
856
+ }
857
+
858
+ if (clusterIdxs.length >= 2) {
859
+ clusters.push(clusterIdxs.map(i => engrams[i]));
860
+ } else {
861
+ unassigned.add(seedIdx);
862
+ }
863
+ }
864
+
865
+ return clusters;
866
+ }
867
+
868
+ /**
869
+ * Compute the centroid (average embedding) of a cluster.
870
+ */
871
+ private computeCentroid(cluster: Engram[]): number[] {
872
+ const withEmbed = cluster.filter(e => e.embedding && e.embedding.length > 0);
873
+ if (withEmbed.length === 0) return [];
874
+
875
+ const dim = withEmbed[0].embedding!.length;
876
+ const centroid = new Array<number>(dim).fill(0);
877
+ for (const e of withEmbed) {
878
+ for (let i = 0; i < dim; i++) {
879
+ centroid[i] += e.embedding![i];
880
+ }
881
+ }
882
+ for (let i = 0; i < dim; i++) {
883
+ centroid[i] /= withEmbed.length;
884
+ }
885
+ return centroid;
886
+ }
887
+ }