agent-working-memory 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +118 -19
  2. package/dist/adapters/claude-code.d.ts.map +1 -1
  3. package/dist/adapters/claude-code.js +63 -3
  4. package/dist/adapters/claude-code.js.map +1 -1
  5. package/dist/adapters/common.d.ts.map +1 -1
  6. package/dist/adapters/common.js +329 -302
  7. package/dist/adapters/common.js.map +1 -1
  8. package/dist/api/routes.d.ts.map +1 -1
  9. package/dist/api/routes.js +31 -8
  10. package/dist/api/routes.js.map +1 -1
  11. package/dist/cli/migrate.js +29 -29
  12. package/dist/cli.js +82 -2
  13. package/dist/cli.js.map +1 -1
  14. package/dist/coordination/circuit-breaker.js +23 -23
  15. package/dist/coordination/routes.d.ts.map +1 -1
  16. package/dist/coordination/routes.js +174 -170
  17. package/dist/coordination/routes.js.map +1 -1
  18. package/dist/core/embeddings.d.ts.map +1 -1
  19. package/dist/core/embeddings.js +3 -0
  20. package/dist/core/embeddings.js.map +1 -1
  21. package/dist/core/entity-extract.d.ts +3 -0
  22. package/dist/core/entity-extract.d.ts.map +1 -0
  23. package/dist/core/entity-extract.js +47 -0
  24. package/dist/core/entity-extract.js.map +1 -0
  25. package/dist/core/salience.d.ts.map +1 -1
  26. package/dist/core/salience.js +14 -2
  27. package/dist/core/salience.js.map +1 -1
  28. package/dist/core/whoami.d.ts +24 -0
  29. package/dist/core/whoami.d.ts.map +1 -0
  30. package/dist/core/whoami.js +66 -0
  31. package/dist/core/whoami.js.map +1 -0
  32. package/dist/core/write-pipeline.d.ts +9 -0
  33. package/dist/core/write-pipeline.d.ts.map +1 -1
  34. package/dist/core/write-pipeline.js +109 -68
  35. package/dist/core/write-pipeline.js.map +1 -1
  36. package/dist/core/write-telemetry.d.ts +33 -0
  37. package/dist/core/write-telemetry.d.ts.map +1 -0
  38. package/dist/core/write-telemetry.js +110 -0
  39. package/dist/core/write-telemetry.js.map +1 -0
  40. package/dist/engine/activation.d.ts +22 -12
  41. package/dist/engine/activation.d.ts.map +1 -1
  42. package/dist/engine/activation.js +133 -17
  43. package/dist/engine/activation.js.map +1 -1
  44. package/dist/engine/consolidation-scheduler.d.ts +1 -1
  45. package/dist/engine/consolidation-scheduler.js +1 -1
  46. package/dist/engine/consolidation.d.ts +1 -0
  47. package/dist/engine/consolidation.d.ts.map +1 -1
  48. package/dist/engine/consolidation.js +18 -0
  49. package/dist/engine/consolidation.js.map +1 -1
  50. package/dist/engine/eval.d.ts.map +1 -1
  51. package/dist/engine/eval.js +5 -1
  52. package/dist/engine/eval.js.map +1 -1
  53. package/dist/index.js +21 -2
  54. package/dist/index.js.map +1 -1
  55. package/dist/mcp.d.ts +2 -1
  56. package/dist/mcp.d.ts.map +1 -1
  57. package/dist/mcp.js +212 -97
  58. package/dist/mcp.js.map +1 -1
  59. package/dist/onboard/index.d.ts +68 -0
  60. package/dist/onboard/index.d.ts.map +1 -0
  61. package/dist/onboard/index.js +265 -0
  62. package/dist/onboard/index.js.map +1 -0
  63. package/dist/recipes/index.d.ts +57 -0
  64. package/dist/recipes/index.d.ts.map +1 -0
  65. package/dist/recipes/index.js +81 -0
  66. package/dist/recipes/index.js.map +1 -0
  67. package/dist/storage/pglite-schema.d.ts.map +1 -1
  68. package/dist/storage/pglite-schema.js +170 -143
  69. package/dist/storage/pglite-schema.js.map +1 -1
  70. package/dist/storage/pglite.d.ts +5 -0
  71. package/dist/storage/pglite.d.ts.map +1 -1
  72. package/dist/storage/pglite.js +180 -138
  73. package/dist/storage/pglite.js.map +1 -1
  74. package/dist/storage/postgres.d.ts +5 -0
  75. package/dist/storage/postgres.d.ts.map +1 -1
  76. package/dist/storage/postgres.js +44 -2
  77. package/dist/storage/postgres.js.map +1 -1
  78. package/dist/storage/sqlite.d.ts +9 -0
  79. package/dist/storage/sqlite.d.ts.map +1 -1
  80. package/dist/storage/sqlite.js +394 -326
  81. package/dist/storage/sqlite.js.map +1 -1
  82. package/dist/types/engram.d.ts +14 -0
  83. package/dist/types/engram.d.ts.map +1 -1
  84. package/dist/types/engram.js.map +1 -1
  85. package/dist/version.d.ts +2 -0
  86. package/dist/version.d.ts.map +1 -0
  87. package/dist/version.js +27 -0
  88. package/dist/version.js.map +1 -0
  89. package/package.json +9 -1
  90. package/src/adapters/claude-code.ts +66 -3
  91. package/src/adapters/common.ts +538 -511
  92. package/src/api/index.ts +3 -3
  93. package/src/api/routes.ts +999 -970
  94. package/src/cli/migrate.ts +307 -307
  95. package/src/cli.ts +77 -2
  96. package/src/coordination/circuit-breaker.ts +83 -83
  97. package/src/coordination/failure-modes.ts +50 -50
  98. package/src/coordination/routes.ts +2155 -2150
  99. package/src/core/decay.ts +63 -63
  100. package/src/core/embeddings.ts +113 -110
  101. package/src/core/entity-extract.ts +47 -0
  102. package/src/core/index.ts +5 -5
  103. package/src/core/logger.ts +36 -36
  104. package/src/core/ml-worker-entry.ts +194 -194
  105. package/src/core/ml-worker.ts +281 -281
  106. package/src/core/query-expander.ts +122 -122
  107. package/src/core/reranker.ts +119 -119
  108. package/src/core/salience.ts +529 -514
  109. package/src/core/whoami.ts +92 -0
  110. package/src/core/write-pipeline.ts +60 -8
  111. package/src/core/write-telemetry.ts +131 -0
  112. package/src/engine/activation.ts +1468 -1369
  113. package/src/engine/confidence.ts +120 -120
  114. package/src/engine/consolidation-scheduler.ts +242 -242
  115. package/src/engine/consolidation.ts +887 -869
  116. package/src/engine/eval.ts +107 -102
  117. package/src/engine/eviction.ts +101 -101
  118. package/src/engine/index.ts +8 -8
  119. package/src/engine/retraction.ts +366 -366
  120. package/src/engine/staging.ts +74 -74
  121. package/src/index.ts +248 -226
  122. package/src/mcp.ts +1341 -1211
  123. package/src/onboard/index.ts +298 -0
  124. package/src/recipes/index.ts +125 -0
  125. package/src/storage/index.ts +3 -3
  126. package/src/storage/pglite-schema.ts +193 -166
  127. package/src/storage/pglite.ts +1420 -1372
  128. package/src/storage/postgres.ts +50 -2
  129. package/src/storage/sqlite.ts +1936 -1861
  130. package/src/storage/store.ts +80 -80
  131. package/src/types/agent.ts +67 -67
  132. package/src/types/checkpoint.ts +46 -46
  133. package/src/types/engram.ts +22 -0
  134. package/src/types/eval.ts +100 -100
  135. package/src/types/index.ts +6 -6
  136. package/src/version.ts +26 -0
@@ -1,1861 +1,1936 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * SQLite storage layer — persistence for engrams, associations, and eval events.
5
- *
6
- * Uses better-sqlite3 for synchronous, fast, embedded storage.
7
- * FTS5 provides BM25 full-text search for the activation pipeline.
8
- */
9
-
10
- import Database from 'better-sqlite3';
11
- import { randomUUID } from 'node:crypto';
12
- import { cosineSimilarity } from '../core/embeddings.js';
13
- import type {
14
- Engram, EngramCreate, EngramStage, Association, AssociationType,
15
- SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
16
- RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
17
- ConsciousState, AutoCheckpoint, CheckpointRow,
18
- } from '../types/index.js';
19
-
20
- /** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
21
- function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
22
- if (buf instanceof ArrayBuffer) return new Float32Array(buf);
23
- // Node Buffer may share an underlying ArrayBuffer — slice to the exact region
24
- const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
25
- return new Float32Array(ab);
26
- }
27
-
28
- const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
29
- surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
30
- };
31
-
32
- /**
33
- * In-memory slim entry — the minimum data the activation pipeline's pre-filter
34
- * pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
35
- * Float32Array conversion on every recall. ~22 bytes overhead per entry plus
36
- * the embedding (~1.5KB), so ~15MB at 10K engrams.
37
- */
38
- type SlimCacheEntry = {
39
- id: string;
40
- agentId: string;
41
- concept: string;
42
- embedding: number[] | null;
43
- stage: EngramStage;
44
- retracted: boolean;
45
- };
46
-
47
- export class EngramStore {
48
- private db: Database.Database;
49
- private walTimer: ReturnType<typeof setInterval> | null = null;
50
-
51
- // Slim cache for the activation pipeline pre-filter. Populated lazily on
52
- // first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
53
- // sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
54
- // (for A/B testing or if a regression appears).
55
- private slimCache: Map<string, SlimCacheEntry> = new Map();
56
- private slimCachePopulated: boolean = false;
57
- private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
58
-
59
- constructor(dbPath: string = 'memory.db') {
60
- this.db = new Database(dbPath);
61
- this.db.pragma('journal_mode = WAL');
62
- this.db.pragma('foreign_keys = ON');
63
- this.db.pragma('busy_timeout = 5000');
64
- this.db.pragma('synchronous = NORMAL');
65
- this.db.pragma('wal_autocheckpoint = 1000');
66
- this.init();
67
- this.startWalCheckpointTimer();
68
- }
69
-
70
- // --- Slim cache management ---
71
-
72
- /** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
73
- private ensureSlimCachePopulated(): void {
74
- if (this.slimCachePopulated || !this.slimCacheEnabled) return;
75
- const rows = this.db.prepare(
76
- 'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
77
- ).all() as any[];
78
- for (const r of rows) {
79
- this.slimCache.set(r.id as string, {
80
- id: r.id as string,
81
- agentId: r.agent_id as string,
82
- concept: r.concept as string,
83
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
84
- stage: r.stage as EngramStage,
85
- retracted: !!r.retracted,
86
- });
87
- }
88
- this.slimCachePopulated = true;
89
- }
90
-
91
- /** Add a new engram to the slim cache. Called from createEngram. */
92
- private cacheAdd(entry: SlimCacheEntry): void {
93
- if (!this.slimCacheEnabled) return;
94
- this.slimCache.set(entry.id, entry);
95
- }
96
-
97
- private cacheUpdateStage(id: string, stage: EngramStage): void {
98
- if (!this.slimCacheEnabled) return;
99
- const e = this.slimCache.get(id);
100
- if (e) e.stage = stage;
101
- }
102
-
103
- private cacheUpdateEmbedding(id: string, embedding: number[]): void {
104
- if (!this.slimCacheEnabled) return;
105
- const e = this.slimCache.get(id);
106
- if (e) e.embedding = embedding;
107
- }
108
-
109
- private cacheRetract(id: string): void {
110
- if (!this.slimCacheEnabled) return;
111
- const e = this.slimCache.get(id);
112
- if (e) e.retracted = true;
113
- }
114
-
115
- private cacheRemove(id: string): void {
116
- if (!this.slimCacheEnabled) return;
117
- this.slimCache.delete(id);
118
- }
119
-
120
- /** Reset cache (used by tests + after timeWarp/bulk operations). */
121
- resetSlimCache(): void {
122
- this.slimCache.clear();
123
- this.slimCachePopulated = false;
124
- }
125
-
126
- /**
127
- * Eager slim-cache populate — public entry point so process startup can warm
128
- * the cache before the first user recall. Otherwise the first recall pays
129
- * a ~600ms one-time SQL fetch + embedding deserialization.
130
- */
131
- warmSlimCache(): void {
132
- this.ensureSlimCachePopulated();
133
- }
134
-
135
- /** Inspect cache state — used for diagnostics + tests. */
136
- getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
137
- return {
138
- populated: this.slimCachePopulated,
139
- size: this.slimCache.size,
140
- enabled: this.slimCacheEnabled,
141
- };
142
- }
143
-
144
- /** Expose the raw database handle for the coordination module. */
145
- getDb(): Database.Database {
146
- return this.db;
147
- }
148
-
149
- /** Run PRAGMA quick_check and return true if DB is healthy. */
150
- integrityCheck(): { ok: boolean; result: string } {
151
- try {
152
- const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
153
- const result = rows[0]?.quick_check ?? 'unknown';
154
- return { ok: result === 'ok', result };
155
- } catch (err) {
156
- return { ok: false, result: (err as Error).message };
157
- }
158
- }
159
-
160
- /** Hot backup using SQLite backup API. Returns the backup path. */
161
- backup(destPath: string): void {
162
- this.db.backup(destPath);
163
- }
164
-
165
- /** Flush WAL to main database file. */
166
- walCheckpoint(): void {
167
- try {
168
- this.db.pragma('wal_checkpoint(TRUNCATE)');
169
- } catch {
170
- // Checkpoint can fail if another connection holds the DB; non-fatal
171
- }
172
- }
173
-
174
- /** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
175
- private startWalCheckpointTimer(): void {
176
- this.walTimer = setInterval(() => {
177
- this.walCheckpoint();
178
- }, 5 * 60 * 1000);
179
- this.walTimer.unref();
180
- }
181
-
182
- /** Stop the WAL checkpoint timer (call before close). */
183
- stopWalCheckpointTimer(): void {
184
- if (this.walTimer) {
185
- clearInterval(this.walTimer);
186
- this.walTimer = null;
187
- }
188
- }
189
-
190
- private init(): void {
191
- this.db.exec(`
192
- CREATE TABLE IF NOT EXISTS engrams (
193
- id TEXT PRIMARY KEY,
194
- agent_id TEXT NOT NULL,
195
- concept TEXT NOT NULL,
196
- content TEXT NOT NULL,
197
- embedding BLOB,
198
- confidence REAL NOT NULL DEFAULT 0.5,
199
- salience REAL NOT NULL DEFAULT 0.5,
200
- access_count INTEGER NOT NULL DEFAULT 0,
201
- last_accessed TEXT NOT NULL,
202
- created_at TEXT NOT NULL,
203
- salience_features TEXT NOT NULL DEFAULT '{}',
204
- reason_codes TEXT NOT NULL DEFAULT '[]',
205
- stage TEXT NOT NULL DEFAULT 'active',
206
- ttl INTEGER,
207
- retracted INTEGER NOT NULL DEFAULT 0,
208
- retracted_by TEXT,
209
- retracted_at TEXT,
210
- tags TEXT NOT NULL DEFAULT '[]',
211
- memory_type TEXT NOT NULL DEFAULT 'unclassified'
212
- );
213
-
214
- CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
215
- CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
216
- CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
217
- CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
218
-
219
- CREATE TABLE IF NOT EXISTS associations (
220
- id TEXT PRIMARY KEY,
221
- from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
222
- to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
223
- weight REAL NOT NULL DEFAULT 0.1,
224
- confidence REAL NOT NULL DEFAULT 0.5,
225
- type TEXT NOT NULL DEFAULT 'hebbian',
226
- activation_count INTEGER NOT NULL DEFAULT 0,
227
- created_at TEXT NOT NULL,
228
- last_activated TEXT NOT NULL
229
- );
230
-
231
- CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
232
- CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
233
- CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
234
-
235
- CREATE TABLE IF NOT EXISTS agents (
236
- id TEXT PRIMARY KEY,
237
- name TEXT NOT NULL,
238
- created_at TEXT NOT NULL,
239
- config TEXT NOT NULL DEFAULT '{}'
240
- );
241
-
242
- -- FTS5 for full-text search (BM25 ranking built in)
243
- CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
244
- concept, content, tags,
245
- content=engrams,
246
- content_rowid=rowid
247
- );
248
-
249
- -- Triggers to keep FTS in sync
250
- CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
251
- INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
252
- END;
253
- CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
254
- INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
255
- END;
256
- CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
257
- INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
258
- INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
259
- END;
260
-
261
- -- Eval event logs
262
- CREATE TABLE IF NOT EXISTS activation_events (
263
- id TEXT PRIMARY KEY,
264
- agent_id TEXT NOT NULL,
265
- timestamp TEXT NOT NULL,
266
- context TEXT NOT NULL,
267
- results_returned INTEGER NOT NULL,
268
- top_score REAL,
269
- latency_ms REAL NOT NULL,
270
- engram_ids TEXT NOT NULL DEFAULT '[]'
271
- );
272
-
273
- CREATE TABLE IF NOT EXISTS staging_events (
274
- engram_id TEXT NOT NULL,
275
- agent_id TEXT NOT NULL,
276
- action TEXT NOT NULL,
277
- resonance_score REAL,
278
- timestamp TEXT NOT NULL,
279
- age_ms INTEGER NOT NULL
280
- );
281
-
282
- CREATE TABLE IF NOT EXISTS retrieval_feedback (
283
- id TEXT PRIMARY KEY,
284
- activation_event_id TEXT,
285
- engram_id TEXT NOT NULL,
286
- useful INTEGER NOT NULL,
287
- context TEXT,
288
- timestamp TEXT NOT NULL
289
- );
290
-
291
- CREATE TABLE IF NOT EXISTS episodes (
292
- id TEXT PRIMARY KEY,
293
- agent_id TEXT NOT NULL,
294
- label TEXT NOT NULL,
295
- embedding BLOB,
296
- engram_count INTEGER NOT NULL DEFAULT 0,
297
- start_time TEXT NOT NULL,
298
- end_time TEXT NOT NULL,
299
- created_at TEXT NOT NULL
300
- );
301
-
302
- CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
303
- CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
304
- `);
305
-
306
- // Migration: add episode_id column if missing
307
- try {
308
- this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
309
- } catch {
310
- this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
311
- }
312
-
313
- // Migration: add task management columns if missing
314
- try {
315
- this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
316
- } catch {
317
- this.db.exec(`
318
- ALTER TABLE engrams ADD COLUMN task_status TEXT;
319
- ALTER TABLE engrams ADD COLUMN task_priority TEXT;
320
- ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
321
- `);
322
- this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
323
- }
324
-
325
- // Migration: add memory_class and supersession columns if missing
326
- try {
327
- this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
328
- } catch {
329
- this.db.exec(`
330
- ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
331
- ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
332
- ALTER TABLE engrams ADD COLUMN supersedes TEXT;
333
- `);
334
- }
335
-
336
- // Migration: add embedding_model for version tracking (prevents drift on model change)
337
- try {
338
- this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
339
- } catch {
340
- this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
341
- }
342
-
343
- // Migration: add conscious_state table for checkpointing
344
- this.db.exec(`
345
- CREATE TABLE IF NOT EXISTS conscious_state (
346
- agent_id TEXT PRIMARY KEY,
347
- last_write_id TEXT,
348
- last_recall_context TEXT,
349
- last_recall_ids TEXT NOT NULL DEFAULT '[]',
350
- last_activity_at TEXT NOT NULL DEFAULT (datetime('now')),
351
- write_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
352
- recall_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
353
- execution_state TEXT,
354
- checkpoint_at TEXT,
355
- last_consolidation_at TEXT,
356
- last_mini_consolidation_at TEXT,
357
- consolidation_cycle_count INTEGER NOT NULL DEFAULT 0,
358
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
359
- )
360
- `);
361
-
362
- // Migration: add consolidation_cycle_count if missing (existing DBs)
363
- try {
364
- this.db.exec(`ALTER TABLE conscious_state ADD COLUMN consolidation_cycle_count INTEGER NOT NULL DEFAULT 0`);
365
- } catch { /* column already exists */ }
366
-
367
- // Migration: add memory_type column if missing
368
- try {
369
- this.db.prepare('SELECT memory_type FROM engrams LIMIT 0').get();
370
- } catch {
371
- this.db.exec(`ALTER TABLE engrams ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'unclassified'`);
372
- }
373
-
374
- // Migration (0.8 Cluster A): sequence column for story-time / chronology.
375
- // Nullable existing engrams stay NULL. Partial index keeps the cost low.
376
- try {
377
- this.db.prepare('SELECT sequence FROM engrams LIMIT 0').get();
378
- } catch {
379
- this.db.exec(`ALTER TABLE engrams ADD COLUMN sequence INTEGER`);
380
- this.db.exec(
381
- `CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence
382
- ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL`
383
- );
384
- }
385
-
386
- // Migration (0.8 Cluster A): references_json column for typed cross-record
387
- // links. Wired through HTTP in Cluster D — schema slot only here.
388
- try {
389
- this.db.prepare('SELECT references_json FROM engrams LIMIT 0').get();
390
- } catch {
391
- this.db.exec(`ALTER TABLE engrams ADD COLUMN references_json TEXT`);
392
- }
393
- }
394
-
395
- // --- Engram CRUD ---
396
-
397
- createEngram(input: EngramCreate): Engram {
398
- const now = new Date().toISOString();
399
- const id = randomUUID();
400
- const embeddingBlob = input.embedding
401
- ? Buffer.from(new Float32Array(input.embedding).buffer)
402
- : null;
403
-
404
- this.db.prepare(`
405
- INSERT INTO engrams (id, agent_id, concept, content, embedding, confidence, salience,
406
- access_count, last_accessed, created_at, salience_features, reason_codes, stage, tags, episode_id,
407
- ttl, memory_class, supersedes, task_status, task_priority, blocked_by, memory_type,
408
- sequence, references_json)
409
- VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
410
- `).run(
411
- id, input.agentId, input.concept, input.content, embeddingBlob,
412
- input.confidence ?? 0.5,
413
- input.salience ?? 0.5,
414
- now, now,
415
- JSON.stringify(input.salienceFeatures ?? DEFAULT_SALIENCE_FEATURES),
416
- JSON.stringify(input.reasonCodes ?? []),
417
- JSON.stringify(input.tags ?? []),
418
- input.episodeId ?? null,
419
- input.ttl ?? null,
420
- input.memoryClass ?? 'working',
421
- input.supersedes ?? null,
422
- input.taskStatus ?? null,
423
- input.taskPriority ?? null,
424
- input.blockedBy ?? null,
425
- input.memoryType ?? 'unclassified',
426
- input.sequence ?? null,
427
- input.references && input.references.length > 0
428
- ? JSON.stringify(input.references) : null,
429
- );
430
-
431
- // Add to slim cache (skip if not yet populated — first slim fetch will load it)
432
- if (this.slimCachePopulated) {
433
- this.cacheAdd({
434
- id,
435
- agentId: input.agentId,
436
- concept: input.concept,
437
- embedding: input.embedding ?? null,
438
- stage: 'active',
439
- retracted: false,
440
- });
441
- }
442
-
443
- return this.getEngram(id)!;
444
- }
445
-
446
- getEngram(id: string): Engram | null {
447
- const row = this.db.prepare('SELECT * FROM engrams WHERE id = ?').get(id) as any;
448
- return row ? this.rowToEngram(row) : null;
449
- }
450
-
451
- getEngramsByAgent(agentId: string, stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
452
- let query = 'SELECT * FROM engrams WHERE agent_id = ?';
453
- const params: any[] = [agentId];
454
-
455
- if (stage) {
456
- query += ' AND stage = ?';
457
- params.push(stage);
458
- }
459
- if (!includeRetracted) {
460
- query += ' AND retracted = 0';
461
- }
462
-
463
- return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
464
- }
465
-
466
- /**
467
- * Slim variant that returns only (id, concept, embedding) — the minimum needed
468
- * for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
469
- * survival check). Avoids materializing the content blob, tag JSON, salience
470
- * features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
471
- *
472
- * Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
473
- * engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
474
- * of that is row materialization of fields we don't read in the filter pass.
475
- */
476
- getEngramsByAgentSlim(
477
- agentId: string,
478
- stage?: EngramStage,
479
- includeRetracted: boolean = false
480
- ): Array<{ id: string; concept: string; embedding: number[] | null }> {
481
- if (this.slimCacheEnabled) {
482
- this.ensureSlimCachePopulated();
483
- const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
484
- for (const entry of this.slimCache.values()) {
485
- if (entry.agentId !== agentId) continue;
486
- if (stage && entry.stage !== stage) continue;
487
- if (!includeRetracted && entry.retracted) continue;
488
- result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
489
- }
490
- return result;
491
- }
492
- // Cache disabled — fall back to direct SQL
493
- let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
494
- const params: any[] = [agentId];
495
-
496
- if (stage) {
497
- query += ' AND stage = ?';
498
- params.push(stage);
499
- }
500
- if (!includeRetracted) {
501
- query += ' AND retracted = 0';
502
- }
503
-
504
- return (this.db.prepare(query).all(...params) as any[]).map(r => ({
505
- id: r.id as string,
506
- concept: r.concept as string,
507
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
508
- }));
509
- }
510
-
511
- /** Slim variant for multi-agent (workspace-scoped) pre-filter. */
512
- getEngramsByAgentsSlim(
513
- agentIds: string[],
514
- stage?: EngramStage,
515
- includeRetracted: boolean = false
516
- ): Array<{ id: string; concept: string; embedding: number[] | null }> {
517
- if (agentIds.length === 0) return [];
518
- if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
519
-
520
- if (this.slimCacheEnabled) {
521
- this.ensureSlimCachePopulated();
522
- const agentSet = new Set(agentIds);
523
- const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
524
- for (const entry of this.slimCache.values()) {
525
- if (!agentSet.has(entry.agentId)) continue;
526
- if (stage && entry.stage !== stage) continue;
527
- if (!includeRetracted && entry.retracted) continue;
528
- result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
529
- }
530
- return result;
531
- }
532
-
533
- const placeholders = agentIds.map(() => '?').join(',');
534
- let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
535
- const params: any[] = [...agentIds];
536
-
537
- if (stage) {
538
- query += ' AND stage = ?';
539
- params.push(stage);
540
- }
541
- if (!includeRetracted) {
542
- query += ' AND retracted = 0';
543
- }
544
-
545
- return (this.db.prepare(query).all(...params) as any[]).map(r => ({
546
- id: r.id as string,
547
- concept: r.concept as string,
548
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
549
- }));
550
- }
551
-
552
- /**
553
- * Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
554
- * only the survivors that need deep scoring. Chunks IN-clause queries to stay
555
- * under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
556
- */
557
- getEngramsByIds(ids: string[]): Engram[] {
558
- if (ids.length === 0) return [];
559
- const CHUNK = 800;
560
- const result: Engram[] = [];
561
- for (let i = 0; i < ids.length; i += CHUNK) {
562
- const chunk = ids.slice(i, i + CHUNK);
563
- const placeholders = chunk.map(() => '?').join(',');
564
- const rows = this.db.prepare(
565
- `SELECT * FROM engrams WHERE id IN (${placeholders})`
566
- ).all(...chunk) as any[];
567
- for (const r of rows) result.push(this.rowToEngram(r));
568
- }
569
- return result;
570
- }
571
-
572
- /**
573
- * Get engrams across multiple agents (workspace-scoped recall).
574
- * Used when workspace mode is enabled for hive memory sharing.
575
- */
576
- getEngramsByAgents(agentIds: string[], stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
577
- if (agentIds.length === 0) return [];
578
- if (agentIds.length === 1) return this.getEngramsByAgent(agentIds[0], stage, includeRetracted);
579
-
580
- const placeholders = agentIds.map(() => '?').join(',');
581
- let query = `SELECT * FROM engrams WHERE agent_id IN (${placeholders})`;
582
- const params: any[] = [...agentIds];
583
-
584
- if (stage) {
585
- query += ' AND stage = ?';
586
- params.push(stage);
587
- }
588
- if (!includeRetracted) {
589
- query += ' AND retracted = 0';
590
- }
591
-
592
- return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
593
- }
594
-
595
- /**
596
- * BM25 search across multiple agents (workspace-scoped).
597
- */
598
- searchBM25WithRankMultiAgent(agentIds: string[], query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
599
- if (agentIds.length === 0) return [];
600
- if (agentIds.length === 1) return this.searchBM25WithRank(agentIds[0], query, limit);
601
-
602
- const sanitized = query
603
- .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
604
- .split(/\s+/)
605
- .filter(w => w.length > 1)
606
- .map(w => `"${w}"`)
607
- .join(' OR ');
608
-
609
- if (!sanitized) return [];
610
-
611
- // CTE prefilter see searchBM25WithRank for rationale (567× speedup verified).
612
- try {
613
- const placeholders = agentIds.map(() => '?').join(',');
614
- const innerLimit = Math.max(limit * 5, 50);
615
- const rows = this.db.prepare(`
616
- WITH top_fts AS (
617
- SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
618
- )
619
- SELECT e.*, top_fts.rank FROM top_fts
620
- JOIN engrams e ON e.rowid = top_fts.rowid
621
- WHERE e.agent_id IN (${placeholders}) AND e.retracted = 0
622
- ORDER BY top_fts.rank
623
- LIMIT ?
624
- `).all(sanitized, innerLimit, ...agentIds, limit) as any[];
625
-
626
- return rows.map(r => ({
627
- engram: this.rowToEngram(r),
628
- bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
629
- }));
630
- } catch {
631
- return [];
632
- }
633
- }
634
-
635
- /**
636
- * Get all distinct agent IDs that share a workspace (requires coord_agents table).
637
- * Returns just the queried agentId if coordination tables don't exist.
638
- */
639
- getWorkspaceAgentIds(agentId: string, workspace: string): string[] {
640
- try {
641
- // Return agent names (not UUIDs) — engrams.agent_id uses name strings
642
- const rows = this.db.prepare(
643
- `SELECT DISTINCT name FROM coord_agents WHERE workspace = ? AND status != 'dead'`
644
- ).all(workspace) as Array<{ name: string }>;
645
- const names = rows.map(r => r.name);
646
- // Ensure the querying agent is always included
647
- if (!names.includes(agentId)) names.push(agentId);
648
- return names;
649
- } catch {
650
- // No coordination tables — fall back to single agent
651
- return [agentId];
652
- }
653
- }
654
-
655
- /**
656
- * Touch an engram: increment access count, update last_accessed, and
657
- * nudge confidence upward. Each retrieval is weak evidence the memory
658
- * is useful bounded so only explicit feedback can push confidence
659
- * above 0.85. Diminishing returns: first accesses matter most.
660
- *
661
- * Boost: +0.02 per access, scaled by 1/sqrt(accessCount+1), capped at 0.85.
662
- */
663
- touchEngram(id: string): void {
664
- this.db.prepare(`
665
- UPDATE engrams
666
- SET access_count = access_count + 1,
667
- last_accessed = ?,
668
- confidence = MIN(0.85, confidence + 0.02 / (1.0 + sqrt(access_count)))
669
- WHERE id = ?
670
- `).run(new Date().toISOString(), id);
671
- }
672
-
673
- updateStage(id: string, stage: EngramStage): void {
674
- this.db.prepare('UPDATE engrams SET stage = ? WHERE id = ?').run(stage, id);
675
- this.cacheUpdateStage(id, stage);
676
- }
677
-
678
- /**
679
- * Replace an engram's content. Used by the fade phase of consolidation
680
- * (Paper 1: storage degradation) to coarsen un-recalled memories without
681
- * deleting them — concept + tags stay intact for tag-based retrieval, but
682
- * BM25 content surface area shrinks.
683
- */
684
- updateContent(id: string, content: string): void {
685
- this.db.prepare('UPDATE engrams SET content = ? WHERE id = ?').run(content, id);
686
- }
687
-
688
- updateConfidence(id: string, confidence: number): void {
689
- this.db.prepare('UPDATE engrams SET confidence = ? WHERE id = ?').run(
690
- Math.max(0, Math.min(1, confidence)), id
691
- );
692
- }
693
-
694
- updateEmbedding(id: string, embedding: number[], modelId?: string): void {
695
- const blob = Buffer.from(new Float32Array(embedding).buffer);
696
- if (modelId) {
697
- this.db.prepare('UPDATE engrams SET embedding = ?, embedding_model = ? WHERE id = ?').run(blob, modelId, id);
698
- } else {
699
- this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
700
- }
701
- this.cacheUpdateEmbedding(id, embedding);
702
- }
703
-
704
- retractEngram(id: string, retractedBy: string | null): void {
705
- this.db.prepare(`
706
- UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
707
- `).run(retractedBy, new Date().toISOString(), id);
708
- this.cacheRetract(id);
709
- }
710
-
711
- deleteEngram(id: string): void {
712
- this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
713
- this.cacheRemove(id);
714
- }
715
-
716
- /**
717
- * Time warp — shift all timestamps backward by ms milliseconds.
718
- * Used for testing time-dependent behavior (decay, forgetting).
719
- * Returns count of records shifted.
720
- */
721
- timeWarp(agentId: string, ms: number): number {
722
- let count = 0;
723
- const shiftSec = Math.round(ms / 1000);
724
- // Shift engram timestamps
725
- const r1 = this.db.prepare(`
726
- UPDATE engrams SET
727
- created_at = datetime(created_at, '-${shiftSec} seconds'),
728
- last_accessed = datetime(last_accessed, '-${shiftSec} seconds')
729
- WHERE agent_id = ?
730
- `).run(agentId);
731
- count += r1.changes;
732
- // Shift association timestamps
733
- const r2 = this.db.prepare(`
734
- UPDATE associations SET
735
- created_at = datetime(created_at, '-${shiftSec} seconds'),
736
- last_activated = datetime(last_activated, '-${shiftSec} seconds')
737
- WHERE from_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
738
- OR to_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
739
- `).run(agentId, agentId);
740
- count += r2.changes;
741
- return count;
742
- }
743
-
744
- // --- Full-text search (BM25) ---
745
-
746
- searchBM25(agentId: string, query: string, limit: number = 10): Engram[] {
747
- return this.searchBM25WithRank(agentId, query, limit).map(r => r.engram);
748
- }
749
-
750
- /**
751
- * Vector similarity top-K search. SQLite has no native vector index, so we
752
- * iterate over the slim cache, compute cosine similarity in-process, and
753
- * return the top-K nearest neighbors. PGlite uses pgvector + ivfflat for
754
- * the same operation in O(log N).
755
- *
756
- * Returns engrams with their cosine distance (0 = identical, 2 = opposite)
757
- * to match the PGlite contract. Activation engine consumes `1 - distance`
758
- * as the cosine similarity score.
759
- */
760
- searchByVector(
761
- agentId: string,
762
- vec: number[],
763
- limit: number = 10,
764
- ): Array<{ engram: Engram; distance: number }> {
765
- if (!vec || vec.length === 0) return [];
766
- // Include both 'active' and 'fading' — faded engrams retain their embedding
767
- // (Paper 1: storage degradation; only content is trimmed) so they should still
768
- // participate in vector recall. Concept + tags + truncated content remain in BM25.
769
- const activeSlim = this.getEngramsByAgentSlim(agentId, 'active', false);
770
- const fadingSlim = this.getEngramsByAgentSlim(agentId, 'fading', false);
771
- const slim = activeSlim.concat(fadingSlim);
772
- const scored: Array<{ id: string; sim: number }> = [];
773
- for (const e of slim) {
774
- if (!e.embedding) continue;
775
- const sim = cosineSimilarity(vec, e.embedding);
776
- scored.push({ id: e.id, sim });
777
- }
778
- scored.sort((a, b) => b.sim - a.sim);
779
- const topIds = scored.slice(0, limit).map(s => s.id);
780
- if (topIds.length === 0) return [];
781
- const engrams = this.getEngramsByIds(topIds);
782
- const engramMap = new Map(engrams.map(e => [e.id, e]));
783
- return scored
784
- .slice(0, limit)
785
- .map(s => ({ engram: engramMap.get(s.id), sim: s.sim }))
786
- .filter((x): x is { engram: Engram; sim: number } => x.engram !== undefined)
787
- .map(x => ({ engram: x.engram, distance: 1 - x.sim }));
788
- }
789
-
790
- /**
791
- * BM25 search returning rank scores alongside engrams.
792
- * FTS5 rank is negative (lower = better match).
793
- * We normalize to 0-1 where higher = better.
794
- */
795
- searchBM25WithRank(agentId: string, query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
796
- // Sanitize query for FTS5: quote each word to prevent column name interpretation
797
- const sanitized = query
798
- .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
799
- .split(/\s+/)
800
- .filter(w => w.length > 1)
801
- .map(w => `"${w}"`)
802
- .join(' OR ');
803
-
804
- if (!sanitized) return [];
805
-
806
- // CTE prefilter: force FTS5 to apply LIMIT before joining engrams.
807
- //
808
- // Why: the obvious query (JOIN engrams_fts ON rowid + WHERE MATCH + ORDER BY rank LIMIT N)
809
- // makes SQLite's planner materialize ALL matching FTS rows joined with engrams
810
- // before applying LIMIT. With wide OR queries on a 17K-engram index, that's
811
- // thousands of row materializations including 1.5KB embedding blobs — measured
812
- // at 3682ms for a 5-term OR query.
813
- //
814
- // The CTE forces FTS5 to LIMIT first (sub-ms), then join only the top-K rowids.
815
- // Same query plan, 567× faster (3682ms → 6ms verified on 17K engrams).
816
- //
817
- // The inner LIMIT (limit * 5) over-fetches because the agent_id + retracted
818
- // filter is applied AFTER the CTE. limit*5 gives enough headroom that filtered
819
- // results still satisfy the outer LIMIT for typical workloads (single agent
820
- // dominant, low retracted rate).
821
- try {
822
- const innerLimit = Math.max(limit * 5, 50);
823
- const rows = this.db.prepare(`
824
- WITH top_fts AS (
825
- SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
826
- )
827
- SELECT e.*, top_fts.rank FROM top_fts
828
- JOIN engrams e ON e.rowid = top_fts.rowid
829
- WHERE e.agent_id = ? AND e.retracted = 0
830
- ORDER BY top_fts.rank
831
- LIMIT ?
832
- `).all(sanitized, innerLimit, agentId, limit) as any[];
833
-
834
- return rows.map(r => ({
835
- engram: this.rowToEngram(r),
836
- // Normalize: rank is negative, more negative = better match.
837
- // |rank| / (1 + |rank|) gives 0-1 where higher = better.
838
- bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
839
- }));
840
- } catch {
841
- return [];
842
- }
843
- }
844
-
845
- // --- Diagnostic search (deterministic, not cognitive) ---
846
-
847
- search(query: SearchQuery): Engram[] {
848
- let sql = 'SELECT * FROM engrams WHERE agent_id = ?';
849
- const params: any[] = [query.agentId];
850
-
851
- if (query.text) {
852
- sql += ' AND (content LIKE ? OR concept LIKE ?)';
853
- params.push(`%${query.text}%`, `%${query.text}%`);
854
- }
855
- if (query.concept) {
856
- sql += ' AND concept = ?';
857
- params.push(query.concept);
858
- }
859
- if (query.stage) {
860
- sql += ' AND stage = ?';
861
- params.push(query.stage);
862
- }
863
- if (query.retracted !== undefined) {
864
- sql += ' AND retracted = ?';
865
- params.push(query.retracted ? 1 : 0);
866
- }
867
-
868
- // ── Tag filters (0.8 Cluster B) ──
869
- // AND-set: merge legacy `tags` + new `tagsAll`. Both apply (intersection).
870
- const allTags = [...(query.tags ?? []), ...(query.tagsAll ?? [])];
871
- if (allTags.length > 0) {
872
- for (const tag of allTags) {
873
- sql += ' AND tags LIKE ?';
874
- params.push(`%"${tag}"%`);
875
- }
876
- }
877
- // OR-set: at least one must match.
878
- if (query.tagsAny && query.tagsAny.length > 0) {
879
- const ors = query.tagsAny.map(() => 'tags LIKE ?').join(' OR ');
880
- sql += ` AND (${ors})`;
881
- for (const tag of query.tagsAny) params.push(`%"${tag}"%`);
882
- }
883
- // NOT-set: none may match.
884
- if (query.tagsNone && query.tagsNone.length > 0) {
885
- const ors = query.tagsNone.map(() => 'tags LIKE ?').join(' OR ');
886
- sql += ` AND NOT (${ors})`;
887
- for (const tag of query.tagsNone) params.push(`%"${tag}"%`);
888
- }
889
-
890
- // ── Sort (0.8 Cluster B) ──
891
- // Default preserves legacy behavior (last_accessed DESC). Map TS camelCase
892
- // to SQL snake_case. `sequence` puts NULLs last via (col IS NULL) trick.
893
- const sortCol = ({
894
- createdAt: 'created_at',
895
- sequence: 'sequence',
896
- salience: 'salience',
897
- confidence: 'confidence',
898
- lastAccessed: 'last_accessed',
899
- } as const)[query.sortBy ?? 'lastAccessed'];
900
- const dir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
901
- if (query.sortBy === 'sequence') {
902
- // NULLs last regardless of direction so engrams without a story-time
903
- // value don't shuffle into the middle of sequenced results.
904
- sql += ` ORDER BY (sequence IS NULL), sequence ${dir}`;
905
- } else {
906
- sql += ` ORDER BY ${sortCol} ${dir}`;
907
- }
908
-
909
- sql += ` LIMIT ? OFFSET ?`;
910
- params.push(query.limit ?? 50, query.offset ?? 0);
911
-
912
- return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
913
- }
914
-
915
- /**
916
- * Get the most recently created engram for an agent (for temporal adjacency edges).
917
- */
918
- getLatestEngram(agentId: string, excludeId?: string): Engram | null {
919
- let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0';
920
- const params: any[] = [agentId];
921
- if (excludeId) {
922
- sql += ' AND id != ?';
923
- params.push(excludeId);
924
- }
925
- sql += ' ORDER BY created_at DESC LIMIT 1';
926
- const row = this.db.prepare(sql).get(...params) as any;
927
- return row ? this.rowToEngram(row) : null;
928
- }
929
-
930
- // --- Task management ---
931
-
932
- updateTaskStatus(id: string, status: TaskStatus): void {
933
- this.db.prepare('UPDATE engrams SET task_status = ? WHERE id = ?').run(status, id);
934
- }
935
-
936
- updateTaskPriority(id: string, priority: TaskPriority): void {
937
- this.db.prepare('UPDATE engrams SET task_priority = ? WHERE id = ?').run(priority, id);
938
- }
939
-
940
- updateBlockedBy(id: string, blockedBy: string | null): void {
941
- this.db.prepare('UPDATE engrams SET blocked_by = ?, task_status = ? WHERE id = ?')
942
- .run(blockedBy, blockedBy ? 'blocked' : 'open', id);
943
- }
944
-
945
- /**
946
- * Get tasks for an agent, optionally filtered by status.
947
- * Results ordered by priority (urgent > high > medium > low), then creation date.
948
- */
949
- getTasks(agentId: string, status?: TaskStatus): Engram[] {
950
- let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND task_status IS NOT NULL AND retracted = 0';
951
- const params: any[] = [agentId];
952
- if (status) {
953
- sql += ' AND task_status = ?';
954
- params.push(status);
955
- }
956
- sql += ` ORDER BY
957
- CASE task_priority
958
- WHEN 'urgent' THEN 0
959
- WHEN 'high' THEN 1
960
- WHEN 'medium' THEN 2
961
- WHEN 'low' THEN 3
962
- ELSE 4
963
- END,
964
- created_at DESC`;
965
- return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
966
- }
967
-
968
- /**
969
- * Get the next actionable task — highest priority that's not blocked or done.
970
- */
971
- getNextTask(agentId: string): Engram | null {
972
- const row = this.db.prepare(`
973
- SELECT * FROM engrams
974
- WHERE agent_id = ? AND task_status IN ('open', 'in_progress') AND retracted = 0
975
- ORDER BY
976
- CASE task_status WHEN 'in_progress' THEN 0 ELSE 1 END,
977
- CASE task_priority
978
- WHEN 'urgent' THEN 0
979
- WHEN 'high' THEN 1
980
- WHEN 'medium' THEN 2
981
- WHEN 'low' THEN 3
982
- ELSE 4
983
- END,
984
- created_at ASC
985
- LIMIT 1
986
- `).get(agentId) as any;
987
- return row ? this.rowToEngram(row) : null;
988
- }
989
-
990
- // --- Supersession ---
991
-
992
- /**
993
- * Mark an engram as superseded by another.
994
- * The old memory stays in the DB (historical) but gets down-ranked in recall.
995
- */
996
- supersedeEngram(oldId: string, newId: string): void {
997
- this.db.prepare('UPDATE engrams SET superseded_by = ? WHERE id = ?').run(newId, oldId);
998
- this.db.prepare('UPDATE engrams SET supersedes = ? WHERE id = ?').run(oldId, newId);
999
- }
1000
-
1001
- /**
1002
- * Find the most recent active engram matching `concept` (case-insensitive,
1003
- * trimmed) and all of `requiredTags`. Excludes superseded and retracted
1004
- * engrams. Used by /memory/supersede Form B (0.8 Cluster D) and by
1005
- * /memory/write's references[] resolution.
1006
- *
1007
- * Returns null if no match.
1008
- */
1009
- findActiveMatchByConcept(
1010
- agentId: string,
1011
- concept: string,
1012
- requiredTags?: string[],
1013
- ): Engram | null {
1014
- let sql = `SELECT * FROM engrams
1015
- WHERE agent_id = ?
1016
- AND LOWER(TRIM(concept)) = LOWER(TRIM(?))
1017
- AND stage = 'active'
1018
- AND retracted = 0
1019
- AND superseded_by IS NULL`;
1020
- const params: any[] = [agentId, concept];
1021
- if (requiredTags && requiredTags.length > 0) {
1022
- for (const tag of requiredTags) {
1023
- sql += ' AND tags LIKE ?';
1024
- params.push(`%"${tag}"%`);
1025
- }
1026
- }
1027
- sql += ' ORDER BY created_at DESC LIMIT 1';
1028
- const row = this.db.prepare(sql).get(...params) as any;
1029
- return row ? this.rowToEngram(row) : null;
1030
- }
1031
-
1032
- /**
1033
- * Check if an engram has been superseded.
1034
- */
1035
- isSuperseded(id: string): boolean {
1036
- const row = this.db.prepare('SELECT superseded_by FROM engrams WHERE id = ?').get(id) as any;
1037
- return row?.superseded_by != null;
1038
- }
1039
-
1040
- updateMemoryClass(id: string, memoryClass: MemoryClass): void {
1041
- this.db.prepare('UPDATE engrams SET memory_class = ? WHERE id = ?').run(memoryClass, id);
1042
- }
1043
-
1044
- updateTags(id: string, tags: string[]): void {
1045
- this.db.prepare('UPDATE engrams SET tags = ? WHERE id = ?').run(JSON.stringify(tags), id);
1046
- }
1047
-
1048
- // --- Associations ---
1049
-
1050
- upsertAssociation(
1051
- fromId: string, toId: string, weight: number,
1052
- type: AssociationType = 'hebbian', confidence: number = 0.5
1053
- ): Association {
1054
- const now = new Date().toISOString();
1055
- const id = randomUUID();
1056
-
1057
- this.db.prepare(`
1058
- INSERT INTO associations (id, from_engram_id, to_engram_id, weight, confidence, type, activation_count, created_at, last_activated)
1059
- VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)
1060
- ON CONFLICT(from_engram_id, to_engram_id) DO UPDATE SET
1061
- weight = ?, confidence = ?, last_activated = ?, activation_count = activation_count + 1
1062
- `).run(id, fromId, toId, weight, confidence, type, now, now, weight, confidence, now);
1063
-
1064
- return this.getAssociation(fromId, toId)!;
1065
- }
1066
-
1067
- getAssociation(fromId: string, toId: string): Association | null {
1068
- const row = this.db.prepare(
1069
- 'SELECT * FROM associations WHERE from_engram_id = ? AND to_engram_id = ?'
1070
- ).get(fromId, toId) as any;
1071
- return row ? this.rowToAssociation(row) : null;
1072
- }
1073
-
1074
- getAssociationsFor(engramId: string): Association[] {
1075
- const rows = this.db.prepare(
1076
- 'SELECT * FROM associations WHERE from_engram_id = ? OR to_engram_id = ?'
1077
- ).all(engramId, engramId);
1078
- return (rows as any[]).map(r => this.rowToAssociation(r));
1079
- }
1080
-
1081
- /**
1082
- * Aggregate association stats per engram — count of edges and sum of weights.
1083
- *
1084
- * Why: the activation scoring loop only uses `associations.length` (Hebbian gate)
1085
- * and `sum of weights` (Hebbian mean + centrality). It doesn't read individual
1086
- * association fields. Returning scalar stats avoids materializing thousands of
1087
- * Association objects.
1088
- *
1089
- * Phase-breakdown spike (2026-05-08, post-0.7.10) showed
1090
- * `getAssociationsForBatch` over ~300 survivors took 222ms (25% of recall floor).
1091
- * Stats-only aggregate via a single GROUP BY drops this to ~20ms.
1092
- *
1093
- * Graph walk still needs full Association rows, but it operates on top-N
1094
- * (~30 candidates) its per-call `getAssociationsFor` is cheap.
1095
- */
1096
- getAssociationStatsForBatch(engramIds: string[]): Map<string, { count: number; sumWeight: number }> {
1097
- const result = new Map<string, { count: number; sumWeight: number }>();
1098
- if (engramIds.length === 0) return result;
1099
-
1100
- const CHUNK = 400;
1101
- for (let i = 0; i < engramIds.length; i += CHUNK) {
1102
- const chunk = engramIds.slice(i, i + CHUNK);
1103
- const placeholders = chunk.map(() => '?').join(',');
1104
- // UNION ALL counts each association once per endpoint that's in the candidate
1105
- // set same semantics as the existing getAssociationsForBatch which buckets
1106
- // associations under both their from and to engram. Self-loops would be
1107
- // double-counted, but they're rare in practice and the prior code handled them
1108
- // identically.
1109
- const rows = this.db.prepare(
1110
- `SELECT id, SUM(cnt) AS count, SUM(sw) AS sum_weight FROM (
1111
- SELECT from_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE from_engram_id IN (${placeholders})
1112
- UNION ALL
1113
- SELECT to_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE to_engram_id IN (${placeholders})
1114
- )
1115
- WHERE id IN (${placeholders})
1116
- GROUP BY id`
1117
- ).all(...chunk, ...chunk, ...chunk) as Array<{ id: string; count: number; sum_weight: number }>;
1118
-
1119
- for (const r of rows) {
1120
- result.set(r.id, { count: r.count, sumWeight: r.sum_weight });
1121
- }
1122
- }
1123
- // Ensure every requested id has an entry (even zero-edge engrams)
1124
- for (const id of engramIds) {
1125
- if (!result.has(id)) result.set(id, { count: 0, sumWeight: 0 });
1126
- }
1127
- return result;
1128
- }
1129
-
1130
- /**
1131
- * Batch variant of getAssociationsFor fetches associations for many engrams
1132
- * in a single query, returning a Map keyed by engram id.
1133
- *
1134
- * Why: per-candidate `getAssociationsFor` calls inside the activation scoring
1135
- * loop are an N+1. Measured at 1300ms for 10K candidates (sub-ms per call but
1136
- * accumulating). One IN-clause query reduces this to ~50ms.
1137
- */
1138
- getAssociationsForBatch(engramIds: string[]): Map<string, Association[]> {
1139
- const result = new Map<string, Association[]>();
1140
- if (engramIds.length === 0) return result;
1141
-
1142
- // SQLite's default SQLITE_LIMIT_VARIABLE_NUMBER is 999. Chunk to stay safely below.
1143
- // We bind each id twice (from + to), so chunks of 400 use 800 placeholders.
1144
- const CHUNK = 400;
1145
- for (let i = 0; i < engramIds.length; i += CHUNK) {
1146
- const chunk = engramIds.slice(i, i + CHUNK);
1147
- const placeholders = chunk.map(() => '?').join(',');
1148
- const rows = this.db.prepare(
1149
- `SELECT * FROM associations
1150
- WHERE from_engram_id IN (${placeholders}) OR to_engram_id IN (${placeholders})`
1151
- ).all(...chunk, ...chunk) as any[];
1152
- for (const r of rows) {
1153
- const a = this.rowToAssociation(r);
1154
- // Bucket by both endpoints getAssociationsFor returns either-direction matches.
1155
- const fromList = result.get(a.fromEngramId) ?? [];
1156
- fromList.push(a);
1157
- result.set(a.fromEngramId, fromList);
1158
- if (a.toEngramId !== a.fromEngramId) {
1159
- const toList = result.get(a.toEngramId) ?? [];
1160
- toList.push(a);
1161
- result.set(a.toEngramId, toList);
1162
- }
1163
- }
1164
- }
1165
- // Ensure every requested id has an entry (even if empty) so callers can
1166
- // .get() without null-checking.
1167
- for (const id of engramIds) {
1168
- if (!result.has(id)) result.set(id, []);
1169
- }
1170
- return result;
1171
- }
1172
-
1173
- getOutgoingAssociations(engramId: string): Association[] {
1174
- const rows = this.db.prepare(
1175
- 'SELECT * FROM associations WHERE from_engram_id = ?'
1176
- ).all(engramId);
1177
- return (rows as any[]).map(r => this.rowToAssociation(r));
1178
- }
1179
-
1180
- countAssociationsFor(engramId: string): number {
1181
- const row = this.db.prepare(
1182
- 'SELECT COUNT(*) as count FROM associations WHERE from_engram_id = ?'
1183
- ).get(engramId) as any;
1184
- return row.count;
1185
- }
1186
-
1187
- getWeakestAssociation(engramId: string): Association | null {
1188
- const row = this.db.prepare(
1189
- 'SELECT * FROM associations WHERE from_engram_id = ? ORDER BY weight ASC LIMIT 1'
1190
- ).get(engramId) as any;
1191
- return row ? this.rowToAssociation(row) : null;
1192
- }
1193
-
1194
- deleteAssociation(id: string): void {
1195
- this.db.prepare('DELETE FROM associations WHERE id = ?').run(id);
1196
- }
1197
-
1198
- getAllAssociations(agentId: string): Association[] {
1199
- const rows = this.db.prepare(`
1200
- SELECT a.* FROM associations a
1201
- JOIN engrams e ON a.from_engram_id = e.id
1202
- WHERE e.agent_id = ?
1203
- `).all(agentId);
1204
- return (rows as any[]).map(r => this.rowToAssociation(r));
1205
- }
1206
-
1207
- // --- Eviction ---
1208
-
1209
- getEvictionCandidates(agentId: string, limit: number): Engram[] {
1210
- // Lowest combined score: low salience + low access + low confidence + oldest
1211
- const rows = this.db.prepare(`
1212
- SELECT * FROM engrams
1213
- WHERE agent_id = ? AND stage = 'active' AND retracted = 0
1214
- ORDER BY (salience * 0.3 + confidence * 0.3 + (CAST(access_count AS REAL) / (access_count + 5)) * 0.2 +
1215
- (1.0 / (1.0 + (julianday('now') - julianday(last_accessed)))) * 0.2) ASC
1216
- LIMIT ?
1217
- `).all(agentId, limit) as any[];
1218
- return rows.map(r => this.rowToEngram(r));
1219
- }
1220
-
1221
- getActiveCount(agentId: string): number {
1222
- const row = this.db.prepare(
1223
- "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'active'"
1224
- ).get(agentId) as any;
1225
- return row.count;
1226
- }
1227
-
1228
- getStagingCount(agentId: string): number {
1229
- const row = this.db.prepare(
1230
- "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'staging'"
1231
- ).get(agentId) as any;
1232
- return row.count;
1233
- }
1234
-
1235
- // --- Staging buffer ---
1236
-
1237
- getExpiredStaging(): Engram[] {
1238
- const now = Date.now();
1239
- const rows = this.db.prepare(`
1240
- SELECT * FROM engrams WHERE stage = 'staging' AND ttl IS NOT NULL
1241
- `).all() as any[];
1242
-
1243
- return rows
1244
- .map(r => this.rowToEngram(r))
1245
- .filter(e => e.ttl && (e.createdAt.getTime() + e.ttl) < now);
1246
- }
1247
-
1248
- // --- Eval event logging ---
1249
-
1250
- logActivationEvent(event: ActivationEvent): void {
1251
- this.db.prepare(`
1252
- INSERT INTO activation_events (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
1253
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1254
- `).run(
1255
- event.id, event.agentId, event.timestamp.toISOString(),
1256
- event.context, event.resultsReturned, event.topScore,
1257
- event.latencyMs, JSON.stringify(event.engramIds)
1258
- );
1259
- }
1260
-
1261
- logStagingEvent(event: StagingEvent): void {
1262
- this.db.prepare(`
1263
- INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
1264
- VALUES (?, ?, ?, ?, ?, ?)
1265
- `).run(
1266
- event.engramId, event.agentId, event.action,
1267
- event.resonanceScore, event.timestamp.toISOString(), event.ageMs
1268
- );
1269
- }
1270
-
1271
- logRetrievalFeedback(activationEventId: string | null, engramId: string, useful: boolean, context: string): void {
1272
- this.db.prepare(`
1273
- INSERT INTO retrieval_feedback (id, activation_event_id, engram_id, useful, context, timestamp)
1274
- VALUES (?, ?, ?, ?, ?, ?)
1275
- `).run(randomUUID(), activationEventId, engramId, useful ? 1 : 0, context, new Date().toISOString());
1276
- }
1277
-
1278
- // --- Eval metrics queries ---
1279
-
1280
- getRetrievalPrecision(agentId: string, windowHours: number = 24): number {
1281
- const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1282
- const row = this.db.prepare(`
1283
- SELECT
1284
- COUNT(CASE WHEN useful = 1 THEN 1 END) as useful_count,
1285
- COUNT(*) as total_count
1286
- FROM retrieval_feedback rf
1287
- LEFT JOIN activation_events ae ON rf.activation_event_id = ae.id
1288
- JOIN engrams e ON rf.engram_id = e.id
1289
- WHERE e.agent_id = ? AND rf.timestamp > ?
1290
- `).get(agentId, since) as any;
1291
-
1292
- return row.total_count > 0 ? row.useful_count / row.total_count : 0;
1293
- }
1294
-
1295
- getStagingMetrics(agentId: string): { promoted: number; discarded: number; expired: number } {
1296
- const row = this.db.prepare(`
1297
- SELECT
1298
- COUNT(CASE WHEN action = 'promoted' THEN 1 END) as promoted,
1299
- COUNT(CASE WHEN action = 'discarded' THEN 1 END) as discarded,
1300
- COUNT(CASE WHEN action = 'expired' THEN 1 END) as expired
1301
- FROM staging_events WHERE agent_id = ?
1302
- `).get(agentId) as any;
1303
- return { promoted: row.promoted, discarded: row.discarded, expired: row.expired };
1304
- }
1305
-
1306
- getActivationStats(agentId: string, windowHours: number = 24): {
1307
- count: number; avgLatencyMs: number; p95LatencyMs: number;
1308
- } {
1309
- const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1310
- const rows = this.db.prepare(`
1311
- SELECT latency_ms FROM activation_events
1312
- WHERE agent_id = ? AND timestamp > ?
1313
- ORDER BY latency_ms ASC
1314
- `).all(agentId, since) as { latency_ms: number }[];
1315
-
1316
- if (rows.length === 0) return { count: 0, avgLatencyMs: 0, p95LatencyMs: 0 };
1317
-
1318
- const total = rows.reduce((s, r) => s + r.latency_ms, 0);
1319
- const p95Index = Math.min(Math.floor(rows.length * 0.95), rows.length - 1);
1320
- return {
1321
- count: rows.length,
1322
- avgLatencyMs: total / rows.length,
1323
- p95LatencyMs: rows[p95Index].latency_ms,
1324
- };
1325
- }
1326
-
1327
- getConsolidatedCount(agentId: string): number {
1328
- const row = this.db.prepare(
1329
- `SELECT COUNT(*) as cnt FROM engrams WHERE agent_id = ? AND stage = 'consolidated'`
1330
- ).get(agentId) as any;
1331
- return row.cnt;
1332
- }
1333
-
1334
- // --- Helpers ---
1335
-
1336
- private rowToEngram(row: any): Engram {
1337
- return {
1338
- id: row.id,
1339
- agentId: row.agent_id,
1340
- concept: row.concept,
1341
- content: row.content,
1342
- embedding: row.embedding
1343
- ? Array.from(bufferToFloat32Array(row.embedding))
1344
- : null,
1345
- confidence: row.confidence,
1346
- salience: row.salience,
1347
- accessCount: row.access_count,
1348
- lastAccessed: new Date(row.last_accessed),
1349
- createdAt: new Date(row.created_at),
1350
- salienceFeatures: JSON.parse(row.salience_features || '{}'),
1351
- reasonCodes: JSON.parse(row.reason_codes || '[]'),
1352
- stage: row.stage as EngramStage,
1353
- ttl: row.ttl,
1354
- retracted: !!row.retracted,
1355
- retractedBy: row.retracted_by,
1356
- retractedAt: row.retracted_at ? new Date(row.retracted_at) : null,
1357
- tags: JSON.parse(row.tags),
1358
- episodeId: row.episode_id ?? null,
1359
- memoryClass: (row.memory_class ?? 'working') as MemoryClass,
1360
- memoryType: (row.memory_type ?? 'unclassified') as MemoryType,
1361
- supersededBy: row.superseded_by ?? null,
1362
- supersedes: row.supersedes ?? null,
1363
- taskStatus: row.task_status ?? null,
1364
- taskPriority: row.task_priority ?? null,
1365
- blockedBy: row.blocked_by ?? null,
1366
- sequence: row.sequence == null ? null : Number(row.sequence),
1367
- references: row.references_json ? JSON.parse(row.references_json) : null,
1368
- };
1369
- }
1370
-
1371
- private rowToAssociation(row: any): Association {
1372
- return {
1373
- id: row.id,
1374
- fromEngramId: row.from_engram_id,
1375
- toEngramId: row.to_engram_id,
1376
- weight: row.weight,
1377
- confidence: row.confidence ?? 0.5,
1378
- type: row.type as AssociationType,
1379
- activationCount: row.activation_count ?? 0,
1380
- createdAt: new Date(row.created_at),
1381
- lastActivated: new Date(row.last_activated),
1382
- };
1383
- }
1384
-
1385
- // --- Episodes ---
1386
-
1387
- createEpisode(input: { agentId: string; label: string; embedding?: number[] }): Episode {
1388
- const now = new Date().toISOString();
1389
- const id = randomUUID();
1390
- const embeddingBlob = input.embedding
1391
- ? Buffer.from(new Float32Array(input.embedding).buffer)
1392
- : null;
1393
-
1394
- this.db.prepare(`
1395
- INSERT INTO episodes (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
1396
- VALUES (?, ?, ?, ?, 0, ?, ?, ?)
1397
- `).run(id, input.agentId, input.label, embeddingBlob, now, now, now);
1398
-
1399
- return this.getEpisode(id)!;
1400
- }
1401
-
1402
- getEpisode(id: string): Episode | null {
1403
- const row = this.db.prepare('SELECT * FROM episodes WHERE id = ?').get(id) as any;
1404
- return row ? this.rowToEpisode(row) : null;
1405
- }
1406
-
1407
- getEpisodesByAgent(agentId: string): Episode[] {
1408
- const rows = this.db.prepare(
1409
- 'SELECT * FROM episodes WHERE agent_id = ? ORDER BY end_time DESC'
1410
- ).all(agentId) as any[];
1411
- return rows.map(r => this.rowToEpisode(r));
1412
- }
1413
-
1414
- getActiveEpisode(agentId: string, windowMs: number = 3600_000): Episode | null {
1415
- // Find most recent episode that ended within the time window
1416
- const cutoff = new Date(Date.now() - windowMs).toISOString();
1417
- const row = this.db.prepare(`
1418
- SELECT * FROM episodes WHERE agent_id = ? AND end_time > ?
1419
- ORDER BY end_time DESC LIMIT 1
1420
- `).get(agentId, cutoff) as any;
1421
- return row ? this.rowToEpisode(row) : null;
1422
- }
1423
-
1424
- addEngramToEpisode(engramId: string, episodeId: string): void {
1425
- this.db.prepare('UPDATE engrams SET episode_id = ? WHERE id = ?').run(episodeId, engramId);
1426
- this.db.prepare(`
1427
- UPDATE episodes SET
1428
- engram_count = engram_count + 1,
1429
- end_time = MAX(end_time, ?)
1430
- WHERE id = ?
1431
- `).run(new Date().toISOString(), episodeId);
1432
- }
1433
-
1434
- getEngramsByEpisode(episodeId: string): Engram[] {
1435
- const rows = this.db.prepare(
1436
- 'SELECT * FROM engrams WHERE episode_id = ? AND retracted = 0 ORDER BY created_at ASC'
1437
- ).all(episodeId) as any[];
1438
- return rows.map(r => this.rowToEngram(r));
1439
- }
1440
-
1441
- updateEpisodeEmbedding(id: string, embedding: number[]): void {
1442
- const blob = Buffer.from(new Float32Array(embedding).buffer);
1443
- this.db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(blob, id);
1444
- }
1445
-
1446
- getEpisodeCount(agentId: string): number {
1447
- const row = this.db.prepare(
1448
- 'SELECT COUNT(*) as cnt FROM episodes WHERE agent_id = ?'
1449
- ).get(agentId) as any;
1450
- return row.cnt;
1451
- }
1452
-
1453
- private rowToEpisode(row: any): Episode {
1454
- return {
1455
- id: row.id,
1456
- agentId: row.agent_id,
1457
- label: row.label,
1458
- embedding: row.embedding
1459
- ? Array.from(bufferToFloat32Array(row.embedding))
1460
- : null,
1461
- engramCount: row.engram_count,
1462
- startTime: new Date(row.start_time),
1463
- endTime: new Date(row.end_time),
1464
- createdAt: new Date(row.created_at),
1465
- };
1466
- }
1467
-
1468
- /**
1469
- * Find engrams whose tags contain any of the given tag values.
1470
- * Used for entity-bridge retrieval: given entity tags from top results,
1471
- * find other engrams mentioning the same entities.
1472
- */
1473
- findEngramsByTags(agentId: string, tags: string[], excludeIds?: Set<string>): Engram[] {
1474
- if (tags.length === 0) return [];
1475
-
1476
- // Build OR conditions for tag matching
1477
- const conditions = tags.map(() => 'tags LIKE ?').join(' OR ');
1478
- const params: any[] = [agentId, ...tags.map(t => `%"${t}"%`)];
1479
-
1480
- let sql = `SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0 AND (${conditions})`;
1481
- const rows = this.db.prepare(sql).all(...params) as any[];
1482
-
1483
- const results = rows.map(r => this.rowToEngram(r));
1484
- if (excludeIds) {
1485
- return results.filter(e => !excludeIds.has(e.id));
1486
- }
1487
- return results;
1488
- }
1489
-
1490
- // --- Checkpointing ---
1491
-
1492
- updateAutoCheckpointWrite(agentId: string, engramId: string): void {
1493
- const now = new Date().toISOString();
1494
- this.db.prepare(`
1495
- INSERT INTO conscious_state (agent_id, last_write_id, last_activity_at, write_count_since_consolidation, updated_at)
1496
- VALUES (?, ?, ?, 1, ?)
1497
- ON CONFLICT(agent_id) DO UPDATE SET
1498
- last_write_id = excluded.last_write_id,
1499
- last_activity_at = excluded.last_activity_at,
1500
- write_count_since_consolidation = write_count_since_consolidation + 1,
1501
- updated_at = excluded.updated_at
1502
- `).run(agentId, engramId, now, now);
1503
- }
1504
-
1505
- updateAutoCheckpointRecall(agentId: string, context: string, engramIds: string[]): void {
1506
- const now = new Date().toISOString();
1507
- this.db.prepare(`
1508
- INSERT INTO conscious_state (agent_id, last_recall_context, last_recall_ids, last_activity_at, recall_count_since_consolidation, updated_at)
1509
- VALUES (?, ?, ?, ?, 1, ?)
1510
- ON CONFLICT(agent_id) DO UPDATE SET
1511
- last_recall_context = excluded.last_recall_context,
1512
- last_recall_ids = excluded.last_recall_ids,
1513
- last_activity_at = excluded.last_activity_at,
1514
- recall_count_since_consolidation = recall_count_since_consolidation + 1,
1515
- updated_at = excluded.updated_at
1516
- `).run(agentId, context, JSON.stringify(engramIds), now, now);
1517
- }
1518
-
1519
- touchActivity(agentId: string): void {
1520
- const now = new Date().toISOString();
1521
- this.db.prepare(`
1522
- INSERT INTO conscious_state (agent_id, last_activity_at, updated_at)
1523
- VALUES (?, ?, ?)
1524
- ON CONFLICT(agent_id) DO UPDATE SET
1525
- last_activity_at = excluded.last_activity_at,
1526
- updated_at = excluded.updated_at
1527
- `).run(agentId, now, now);
1528
- }
1529
-
1530
- saveCheckpoint(agentId: string, state: ConsciousState): void {
1531
- const now = new Date().toISOString();
1532
- this.db.prepare(`
1533
- INSERT INTO conscious_state (agent_id, execution_state, checkpoint_at, last_activity_at, updated_at)
1534
- VALUES (?, ?, ?, ?, ?)
1535
- ON CONFLICT(agent_id) DO UPDATE SET
1536
- execution_state = excluded.execution_state,
1537
- checkpoint_at = excluded.checkpoint_at,
1538
- last_activity_at = excluded.last_activity_at,
1539
- updated_at = excluded.updated_at
1540
- `).run(agentId, JSON.stringify(state), now, now, now);
1541
- }
1542
-
1543
- getCheckpoint(agentId: string): CheckpointRow | null {
1544
- const row = this.db.prepare('SELECT * FROM conscious_state WHERE agent_id = ?').get(agentId) as any;
1545
- if (!row) return null;
1546
-
1547
- return {
1548
- agentId: row.agent_id,
1549
- auto: {
1550
- lastWriteId: row.last_write_id ?? null,
1551
- lastRecallContext: row.last_recall_context ?? null,
1552
- lastRecallIds: JSON.parse(row.last_recall_ids || '[]'),
1553
- lastActivityAt: new Date(row.last_activity_at),
1554
- writeCountSinceConsolidation: row.write_count_since_consolidation,
1555
- recallCountSinceConsolidation: row.recall_count_since_consolidation,
1556
- },
1557
- executionState: row.execution_state ? JSON.parse(row.execution_state) : null,
1558
- checkpointAt: row.checkpoint_at ? new Date(row.checkpoint_at) : null,
1559
- lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1560
- lastMiniConsolidationAt: row.last_mini_consolidation_at ? new Date(row.last_mini_consolidation_at) : null,
1561
- updatedAt: new Date(row.updated_at),
1562
- };
1563
- }
1564
-
1565
- markConsolidation(agentId: string, mini: boolean): void {
1566
- const now = new Date().toISOString();
1567
- if (mini) {
1568
- this.db.prepare(`
1569
- UPDATE conscious_state SET last_mini_consolidation_at = ?, updated_at = ? WHERE agent_id = ?
1570
- `).run(now, now, agentId);
1571
- } else {
1572
- this.db.prepare(`
1573
- UPDATE conscious_state SET
1574
- last_consolidation_at = ?,
1575
- last_mini_consolidation_at = ?,
1576
- write_count_since_consolidation = 0,
1577
- recall_count_since_consolidation = 0,
1578
- consolidation_cycle_count = consolidation_cycle_count + 1,
1579
- updated_at = ?
1580
- WHERE agent_id = ?
1581
- `).run(now, now, now, agentId);
1582
- }
1583
- }
1584
-
1585
- getActiveAgents(): Array<{ agentId: string; lastActivityAt: Date; writeCount: number; recallCount: number; lastConsolidationAt: Date | null }> {
1586
- const rows = this.db.prepare('SELECT * FROM conscious_state').all() as any[];
1587
- return rows.map(row => ({
1588
- agentId: row.agent_id,
1589
- lastActivityAt: new Date(row.last_activity_at),
1590
- writeCount: row.write_count_since_consolidation,
1591
- recallCount: row.recall_count_since_consolidation,
1592
- lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1593
- }));
1594
- }
1595
-
1596
- getConsolidationCycleCount(agentId: string): number {
1597
- const row = this.db.prepare(
1598
- 'SELECT consolidation_cycle_count FROM conscious_state WHERE agent_id = ?',
1599
- ).get(agentId) as { consolidation_cycle_count: number } | undefined;
1600
- return row?.consolidation_cycle_count ?? 0;
1601
- }
1602
-
1603
- close(): void {
1604
- this.stopWalCheckpointTimer();
1605
- this.walCheckpoint();
1606
- this.db.close();
1607
- }
1608
-
1609
- /**
1610
- * Run `fn` in a SQL transaction. All writes succeed atomically or roll back
1611
- * on throw. Synchronous (matches better-sqlite3 semantics).
1612
- *
1613
- * Used by 0.8 Cluster D's `/memory/supersede` Form B to wrap find-match +
1614
- * performWrite + supersede + causal-edge + confidence-decay so callers
1615
- * never observe an intermediate state where new engram exists but old
1616
- * isn't yet superseded.
1617
- */
1618
- transaction<T>(fn: () => T): T {
1619
- return this.db.transaction(fn)();
1620
- }
1621
-
1622
- /**
1623
- * Async-aware transaction wrapper. `fn` may await — we hold the SQLite
1624
- * lock across awaits by issuing raw BEGIN/COMMIT/ROLLBACK statements rather
1625
- * than using better-sqlite3's sync `db.transaction()`.
1626
- *
1627
- * Used by Form B (atomic write + supersede) where the body calls
1628
- * async store methods through the IEngramStore contract.
1629
- */
1630
- async withTransaction<T>(fn: () => Promise<T>): Promise<T> {
1631
- this.db.exec('BEGIN');
1632
- try {
1633
- const result = await fn();
1634
- this.db.exec('COMMIT');
1635
- return result;
1636
- } catch (err) {
1637
- try { this.db.exec('ROLLBACK'); } catch { /* best-effort */ }
1638
- throw err;
1639
- }
1640
- }
1641
-
1642
- // ============================================================
1643
- // 0.8 Cluster C — materialized-view + atomic-counter primitives
1644
- // ============================================================
1645
-
1646
- /**
1647
- * For each distinct value of the tag whose key prefix is `tagKeyPrefix`
1648
- * (e.g. `"character="`), return the most-recent active engram. Used by
1649
- * NovelForge's "latest emotional state per character", "latest motif
1650
- * phase per motif", "recent chapter summaries".
1651
- *
1652
- * `sortBy` defaults to `createdAt`. With `"sequence"`, engrams without a
1653
- * sequence are excluded (they have no story-time anchor).
1654
- */
1655
- getLatestByTag(opts: {
1656
- agentId: string;
1657
- tagKeyPrefix: string; // e.g. "character=", "motif=", "chapter="
1658
- scopeTagsAll?: string[]; // optional narrowing (e.g. ["topic=motif-use"])
1659
- retracted?: boolean;
1660
- sortBy?: 'createdAt' | 'sequence';
1661
- limit?: number;
1662
- }): Engram[] {
1663
- let sql = `SELECT * FROM engrams
1664
- WHERE agent_id = ?
1665
- AND retracted = ?
1666
- AND stage = 'active'
1667
- AND tags LIKE ?`;
1668
- const params: any[] = [
1669
- opts.agentId,
1670
- opts.retracted ? 1 : 0,
1671
- `%"${opts.tagKeyPrefix}%`,
1672
- ];
1673
- if (opts.scopeTagsAll && opts.scopeTagsAll.length > 0) {
1674
- for (const t of opts.scopeTagsAll) {
1675
- sql += ' AND tags LIKE ?';
1676
- params.push(`%"${t}"%`);
1677
- }
1678
- }
1679
- if (opts.sortBy === 'sequence') {
1680
- sql += ' AND sequence IS NOT NULL';
1681
- }
1682
- sql += ' ORDER BY ' + (opts.sortBy === 'sequence' ? 'sequence DESC, created_at DESC' : 'created_at DESC');
1683
-
1684
- const rows = this.db.prepare(sql).all(...params) as any[];
1685
- const engrams = rows.map(r => this.rowToEngram(r));
1686
-
1687
- // Group by extracted tag value, take first (already sorted newest first).
1688
- const seen = new Map<string, Engram>();
1689
- for (const e of engrams) {
1690
- const value = this.extractTagValue(e.tags, opts.tagKeyPrefix);
1691
- if (value == null) continue;
1692
- if (!seen.has(value)) seen.set(value, e);
1693
- }
1694
- const out = Array.from(seen.values());
1695
- return opts.limit ? out.slice(0, opts.limit) : out;
1696
- }
1697
-
1698
- /**
1699
- * Filter engrams by tag-set operators and sort by numeric value extracted
1700
- * from a tag prefix (e.g. `"weight="` → numeric value of `weight=8`).
1701
- * Used by NovelForge's "top 40 active promises by weight".
1702
- *
1703
- * Missing or unparseable numeric values sort last.
1704
- */
1705
- getTopBy(opts: {
1706
- agentId: string;
1707
- sortField: string; // tag prefix, e.g. "weight="
1708
- order: 'asc' | 'desc';
1709
- filterTagsAll?: string[];
1710
- filterTagsAny?: string[];
1711
- filterTagsNone?: string[];
1712
- limit?: number;
1713
- retracted?: boolean;
1714
- }): Engram[] {
1715
- // Build the base query inline (mirrors search() but adds the topBy sort).
1716
- let sql = `SELECT * FROM engrams
1717
- WHERE agent_id = ?
1718
- AND retracted = ?
1719
- AND stage = 'active'
1720
- AND tags LIKE ?`;
1721
- const params: any[] = [
1722
- opts.agentId,
1723
- opts.retracted ? 1 : 0,
1724
- `%"${opts.sortField}%`, // must have a tag with this prefix
1725
- ];
1726
-
1727
- if (opts.filterTagsAll && opts.filterTagsAll.length > 0) {
1728
- for (const tag of opts.filterTagsAll) {
1729
- sql += ' AND tags LIKE ?';
1730
- params.push(`%"${tag}"%`);
1731
- }
1732
- }
1733
- if (opts.filterTagsAny && opts.filterTagsAny.length > 0) {
1734
- const ors = opts.filterTagsAny.map(() => 'tags LIKE ?').join(' OR ');
1735
- sql += ` AND (${ors})`;
1736
- for (const tag of opts.filterTagsAny) params.push(`%"${tag}"%`);
1737
- }
1738
- if (opts.filterTagsNone && opts.filterTagsNone.length > 0) {
1739
- const ors = opts.filterTagsNone.map(() => 'tags LIKE ?').join(' OR ');
1740
- sql += ` AND NOT (${ors})`;
1741
- for (const tag of opts.filterTagsNone) params.push(`%"${tag}"%`);
1742
- }
1743
-
1744
- const rows = this.db.prepare(sql).all(...params) as any[];
1745
- const engrams = rows.map(r => this.rowToEngram(r));
1746
-
1747
- // Extract numeric value from the sortField tag; missing or NaN sorts last.
1748
- const valued = engrams.map(e => {
1749
- const raw = this.extractTagValue(e.tags, opts.sortField);
1750
- const n = raw == null ? NaN : Number(raw);
1751
- return { e, n };
1752
- });
1753
- valued.sort((a, b) => {
1754
- const aNaN = Number.isNaN(a.n);
1755
- const bNaN = Number.isNaN(b.n);
1756
- if (aNaN && bNaN) return 0;
1757
- if (aNaN) return 1; // NaN last
1758
- if (bNaN) return -1;
1759
- return opts.order === 'asc' ? a.n - b.n : b.n - a.n;
1760
- });
1761
- const sorted = valued.map(v => v.e);
1762
- return opts.limit ? sorted.slice(0, opts.limit) : sorted;
1763
- }
1764
-
1765
- /**
1766
- * Compute the effective state of an engram from referenced events.
1767
- *
1768
- * - 'superseded' if engram.supersededBy is set
1769
- * - else: scan engrams whose references_json names this engram with a
1770
- * terminal relation type (resolves / subverts / abandons). Take the
1771
- * latest by createdAt. effectiveState = that type.
1772
- * - else: 'active'
1773
- *
1774
- * Returns `null` if target engram not found.
1775
- */
1776
- resolveEffectiveState(targetEngramId: string): {
1777
- engram: Engram;
1778
- effectiveState: 'active' | 'resolved' | 'subverted' | 'abandoned' | 'superseded';
1779
- resolvingEvents: Array<{ id: string; type: string; createdAt: string }>;
1780
- } | null {
1781
- const target = this.getEngram(targetEngramId);
1782
- if (!target) return null;
1783
-
1784
- if (target.supersededBy) {
1785
- return { engram: target, effectiveState: 'superseded', resolvingEvents: [] };
1786
- }
1787
-
1788
- // Find candidate referencing engrams. LIKE-filter first to narrow,
1789
- // then JSON.parse each candidate's references_json to verify the
1790
- // matchEngramId and type. Cheap enough at NovelForge's scale.
1791
- const candidates = this.db.prepare(
1792
- `SELECT id, references_json, created_at FROM engrams
1793
- WHERE agent_id = ?
1794
- AND retracted = 0
1795
- AND stage = 'active'
1796
- AND references_json IS NOT NULL
1797
- AND references_json LIKE ?`
1798
- ).all(target.agentId, `%"${targetEngramId}"%`) as Array<{
1799
- id: string;
1800
- references_json: string;
1801
- created_at: string;
1802
- }>;
1803
-
1804
- const terminalTypes = new Set(['resolves', 'subverts', 'abandons']);
1805
- const resolvingEvents: Array<{ id: string; type: string; createdAt: string }> = [];
1806
- for (const c of candidates) {
1807
- try {
1808
- const refs = JSON.parse(c.references_json) as Array<{ type: string; matchEngramId?: string }>;
1809
- for (const ref of refs) {
1810
- if (ref.matchEngramId === targetEngramId && terminalTypes.has(ref.type)) {
1811
- resolvingEvents.push({ id: c.id, type: ref.type, createdAt: c.created_at });
1812
- }
1813
- }
1814
- } catch { /* malformed references_json skip */ }
1815
- }
1816
-
1817
- if (resolvingEvents.length === 0) {
1818
- return { engram: target, effectiveState: 'active', resolvingEvents: [] };
1819
- }
1820
-
1821
- // Latest event wins.
1822
- resolvingEvents.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
1823
- const latestType = resolvingEvents[0]!.type as 'resolves' | 'subverts' | 'abandons';
1824
- const effectiveState = (
1825
- latestType === 'resolves' ? 'resolved'
1826
- : latestType === 'subverts' ? 'subverted'
1827
- : 'abandoned'
1828
- );
1829
- return { engram: target, effectiveState, resolvingEvents };
1830
- }
1831
-
1832
- /**
1833
- * Atomically allocate the next sequence number for an agent.
1834
- *
1835
- * Uses BEGIN IMMEDIATE to serialize concurrent callers — better-sqlite3's
1836
- * transaction() acquires a RESERVED lock which prevents other writers
1837
- * from racing. Returns MAX(sequence)+1 for the agent, or 1 if no engrams
1838
- * have a sequence yet.
1839
- */
1840
- allocateNextSequence(agentId: string): number {
1841
- return this.transaction(() => {
1842
- const row = this.db.prepare(
1843
- 'SELECT MAX(sequence) AS max_seq FROM engrams WHERE agent_id = ?'
1844
- ).get(agentId) as { max_seq: number | null };
1845
- return (row?.max_seq ?? 0) + 1;
1846
- });
1847
- }
1848
-
1849
- /**
1850
- * Extract the value following a tag prefix, e.g. `["weight=8"]` + `"weight="`
1851
- * `"8"`. Returns null if no tag with that prefix is present.
1852
- */
1853
- private extractTagValue(tags: string[], tagKeyPrefix: string): string | null {
1854
- for (const t of tags) {
1855
- if (t.startsWith(tagKeyPrefix)) {
1856
- return t.slice(tagKeyPrefix.length);
1857
- }
1858
- }
1859
- return null;
1860
- }
1861
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * SQLite storage layer — persistence for engrams, associations, and eval events.
5
+ *
6
+ * Uses better-sqlite3 for synchronous, fast, embedded storage.
7
+ * FTS5 provides BM25 full-text search for the activation pipeline.
8
+ */
9
+
10
+ import Database from 'better-sqlite3';
11
+ import { randomUUID } from 'node:crypto';
12
+ import { cosineSimilarity } from '../core/embeddings.js';
13
+ import type {
14
+ Engram, EngramCreate, EngramStage, Association, AssociationType,
15
+ SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
16
+ RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
17
+ ConsciousState, AutoCheckpoint, CheckpointRow,
18
+ } from '../types/index.js';
19
+
20
+ /** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
21
+ function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
22
+ if (buf instanceof ArrayBuffer) return new Float32Array(buf);
23
+ // Node Buffer may share an underlying ArrayBuffer — slice to the exact region
24
+ const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
25
+ return new Float32Array(ab);
26
+ }
27
+
28
+ const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
29
+ surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
30
+ };
31
+
32
+ /**
33
+ * In-memory slim entry — the minimum data the activation pipeline's pre-filter
34
+ * pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
35
+ * Float32Array conversion on every recall. ~22 bytes overhead per entry plus
36
+ * the embedding (~1.5KB), so ~15MB at 10K engrams.
37
+ */
38
+ type SlimCacheEntry = {
39
+ id: string;
40
+ agentId: string;
41
+ concept: string;
42
+ embedding: number[] | null;
43
+ stage: EngramStage;
44
+ retracted: boolean;
45
+ };
46
+
47
+ export class EngramStore {
48
+ private db: Database.Database;
49
+ private walTimer: ReturnType<typeof setInterval> | null = null;
50
+
51
+ // Slim cache for the activation pipeline pre-filter. Populated lazily on
52
+ // first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
53
+ // sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
54
+ // (for A/B testing or if a regression appears).
55
+ private slimCache: Map<string, SlimCacheEntry> = new Map();
56
+ private slimCachePopulated: boolean = false;
57
+ private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
58
+
59
+ constructor(dbPath: string = 'memory.db') {
60
+ this.db = new Database(dbPath);
61
+ this.db.pragma('journal_mode = WAL');
62
+ this.db.pragma('foreign_keys = ON');
63
+ this.db.pragma('busy_timeout = 5000');
64
+ this.db.pragma('synchronous = NORMAL');
65
+ this.db.pragma('wal_autocheckpoint = 1000');
66
+ this.init();
67
+ this.startWalCheckpointTimer();
68
+ }
69
+
70
+ // --- Slim cache management ---
71
+
72
+ /** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
73
+ private ensureSlimCachePopulated(): void {
74
+ if (this.slimCachePopulated || !this.slimCacheEnabled) return;
75
+ const rows = this.db.prepare(
76
+ 'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
77
+ ).all() as any[];
78
+ for (const r of rows) {
79
+ this.slimCache.set(r.id as string, {
80
+ id: r.id as string,
81
+ agentId: r.agent_id as string,
82
+ concept: r.concept as string,
83
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
84
+ stage: r.stage as EngramStage,
85
+ retracted: !!r.retracted,
86
+ });
87
+ }
88
+ this.slimCachePopulated = true;
89
+ }
90
+
91
+ /** Add a new engram to the slim cache. Called from createEngram. */
92
+ private cacheAdd(entry: SlimCacheEntry): void {
93
+ if (!this.slimCacheEnabled) return;
94
+ this.slimCache.set(entry.id, entry);
95
+ }
96
+
97
+ private cacheUpdateStage(id: string, stage: EngramStage): void {
98
+ if (!this.slimCacheEnabled) return;
99
+ const e = this.slimCache.get(id);
100
+ if (e) e.stage = stage;
101
+ }
102
+
103
+ private cacheUpdateEmbedding(id: string, embedding: number[]): void {
104
+ if (!this.slimCacheEnabled) return;
105
+ const e = this.slimCache.get(id);
106
+ if (e) e.embedding = embedding;
107
+ }
108
+
109
+ private cacheRetract(id: string): void {
110
+ if (!this.slimCacheEnabled) return;
111
+ const e = this.slimCache.get(id);
112
+ if (e) e.retracted = true;
113
+ }
114
+
115
+ private cacheRemove(id: string): void {
116
+ if (!this.slimCacheEnabled) return;
117
+ this.slimCache.delete(id);
118
+ }
119
+
120
+ /** Reset cache (used by tests + after timeWarp/bulk operations). */
121
+ resetSlimCache(): void {
122
+ this.slimCache.clear();
123
+ this.slimCachePopulated = false;
124
+ }
125
+
126
+ /**
127
+ * Eager slim-cache populate — public entry point so process startup can warm
128
+ * the cache before the first user recall. Otherwise the first recall pays
129
+ * a ~600ms one-time SQL fetch + embedding deserialization.
130
+ */
131
+ warmSlimCache(): void {
132
+ this.ensureSlimCachePopulated();
133
+ }
134
+
135
+ /** Inspect cache state — used for diagnostics + tests. */
136
+ getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
137
+ return {
138
+ populated: this.slimCachePopulated,
139
+ size: this.slimCache.size,
140
+ enabled: this.slimCacheEnabled,
141
+ };
142
+ }
143
+
144
+ /** Expose the raw database handle for the coordination module. */
145
+ getDb(): Database.Database {
146
+ return this.db;
147
+ }
148
+
149
+ /** Run PRAGMA quick_check and return true if DB is healthy. */
150
+ integrityCheck(): { ok: boolean; result: string } {
151
+ try {
152
+ const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
153
+ const result = rows[0]?.quick_check ?? 'unknown';
154
+ return { ok: result === 'ok', result };
155
+ } catch (err) {
156
+ return { ok: false, result: (err as Error).message };
157
+ }
158
+ }
159
+
160
+ /** Hot backup using SQLite backup API. Returns the backup path. */
161
+ backup(destPath: string): void {
162
+ this.db.backup(destPath);
163
+ }
164
+
165
+ /** Flush WAL to main database file. */
166
+ walCheckpoint(): void {
167
+ try {
168
+ this.db.pragma('wal_checkpoint(TRUNCATE)');
169
+ } catch {
170
+ // Checkpoint can fail if another connection holds the DB; non-fatal
171
+ }
172
+ }
173
+
174
+ /** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
175
+ private startWalCheckpointTimer(): void {
176
+ this.walTimer = setInterval(() => {
177
+ this.walCheckpoint();
178
+ }, 5 * 60 * 1000);
179
+ this.walTimer.unref();
180
+ }
181
+
182
+ /** Stop the WAL checkpoint timer (call before close). */
183
+ stopWalCheckpointTimer(): void {
184
+ if (this.walTimer) {
185
+ clearInterval(this.walTimer);
186
+ this.walTimer = null;
187
+ }
188
+ }
189
+
190
+ private init(): void {
191
+ this.db.exec(`
192
+ CREATE TABLE IF NOT EXISTS engrams (
193
+ id TEXT PRIMARY KEY,
194
+ agent_id TEXT NOT NULL,
195
+ concept TEXT NOT NULL,
196
+ content TEXT NOT NULL,
197
+ embedding BLOB,
198
+ confidence REAL NOT NULL DEFAULT 0.5,
199
+ salience REAL NOT NULL DEFAULT 0.5,
200
+ access_count INTEGER NOT NULL DEFAULT 0,
201
+ last_accessed TEXT NOT NULL,
202
+ created_at TEXT NOT NULL,
203
+ salience_features TEXT NOT NULL DEFAULT '{}',
204
+ reason_codes TEXT NOT NULL DEFAULT '[]',
205
+ stage TEXT NOT NULL DEFAULT 'active',
206
+ ttl INTEGER,
207
+ retracted INTEGER NOT NULL DEFAULT 0,
208
+ retracted_by TEXT,
209
+ retracted_at TEXT,
210
+ tags TEXT NOT NULL DEFAULT '[]',
211
+ memory_type TEXT NOT NULL DEFAULT 'unclassified'
212
+ );
213
+
214
+ CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
215
+ CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
216
+ CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
217
+ CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
218
+
219
+ CREATE TABLE IF NOT EXISTS associations (
220
+ id TEXT PRIMARY KEY,
221
+ from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
222
+ to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
223
+ weight REAL NOT NULL DEFAULT 0.1,
224
+ confidence REAL NOT NULL DEFAULT 0.5,
225
+ type TEXT NOT NULL DEFAULT 'hebbian',
226
+ activation_count INTEGER NOT NULL DEFAULT 0,
227
+ created_at TEXT NOT NULL,
228
+ last_activated TEXT NOT NULL
229
+ );
230
+
231
+ CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
232
+ CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
233
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
234
+
235
+ CREATE TABLE IF NOT EXISTS agents (
236
+ id TEXT PRIMARY KEY,
237
+ name TEXT NOT NULL,
238
+ created_at TEXT NOT NULL,
239
+ config TEXT NOT NULL DEFAULT '{}'
240
+ );
241
+
242
+ -- FTS5 for full-text search (BM25 ranking built in)
243
+ CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
244
+ concept, content, tags,
245
+ content=engrams,
246
+ content_rowid=rowid
247
+ );
248
+
249
+ -- Triggers to keep FTS in sync
250
+ CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
251
+ INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
252
+ END;
253
+ CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
254
+ INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
255
+ END;
256
+ CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
257
+ INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
258
+ INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
259
+ END;
260
+
261
+ -- Eval event logs
262
+ CREATE TABLE IF NOT EXISTS activation_events (
263
+ id TEXT PRIMARY KEY,
264
+ agent_id TEXT NOT NULL,
265
+ timestamp TEXT NOT NULL,
266
+ context TEXT NOT NULL,
267
+ results_returned INTEGER NOT NULL,
268
+ top_score REAL,
269
+ latency_ms REAL NOT NULL,
270
+ engram_ids TEXT NOT NULL DEFAULT '[]'
271
+ );
272
+
273
+ CREATE TABLE IF NOT EXISTS staging_events (
274
+ engram_id TEXT NOT NULL,
275
+ agent_id TEXT NOT NULL,
276
+ action TEXT NOT NULL,
277
+ resonance_score REAL,
278
+ timestamp TEXT NOT NULL,
279
+ age_ms INTEGER NOT NULL
280
+ );
281
+
282
+ CREATE TABLE IF NOT EXISTS retrieval_feedback (
283
+ id TEXT PRIMARY KEY,
284
+ activation_event_id TEXT,
285
+ engram_id TEXT NOT NULL,
286
+ useful INTEGER NOT NULL,
287
+ context TEXT,
288
+ timestamp TEXT NOT NULL
289
+ );
290
+
291
+ CREATE TABLE IF NOT EXISTS episodes (
292
+ id TEXT PRIMARY KEY,
293
+ agent_id TEXT NOT NULL,
294
+ label TEXT NOT NULL,
295
+ embedding BLOB,
296
+ engram_count INTEGER NOT NULL DEFAULT 0,
297
+ start_time TEXT NOT NULL,
298
+ end_time TEXT NOT NULL,
299
+ created_at TEXT NOT NULL
300
+ );
301
+
302
+ CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
303
+ CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
304
+ `);
305
+
306
+ // Migration: add episode_id column if missing
307
+ try {
308
+ this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
309
+ } catch {
310
+ this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
311
+ }
312
+
313
+ // Migration: add task management columns if missing
314
+ try {
315
+ this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
316
+ } catch {
317
+ this.db.exec(`
318
+ ALTER TABLE engrams ADD COLUMN task_status TEXT;
319
+ ALTER TABLE engrams ADD COLUMN task_priority TEXT;
320
+ ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
321
+ `);
322
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
323
+ }
324
+
325
+ // Migration: add memory_class and supersession columns if missing
326
+ try {
327
+ this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
328
+ } catch {
329
+ this.db.exec(`
330
+ ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
331
+ ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
332
+ ALTER TABLE engrams ADD COLUMN supersedes TEXT;
333
+ `);
334
+ }
335
+
336
+ // Migration: add embedding_model for version tracking (prevents drift on model change)
337
+ try {
338
+ this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
339
+ } catch {
340
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
341
+ }
342
+
343
+ // Entity inverted index + alias table (D9, 2026-07-30) — write-time
344
+ // bookkeeping only; retrieval unchanged until D11. Idempotent CREATEs.
345
+ this.db.exec(`
346
+ CREATE TABLE IF NOT EXISTS entity_mentions (
347
+ entity TEXT NOT NULL,
348
+ engram_id TEXT NOT NULL,
349
+ agent_id TEXT NOT NULL,
350
+ PRIMARY KEY (entity, engram_id)
351
+ );
352
+ CREATE INDEX IF NOT EXISTS idx_entity_mentions_agent ON entity_mentions(agent_id, entity);
353
+ CREATE TABLE IF NOT EXISTS entity_aliases (
354
+ alias TEXT PRIMARY KEY,
355
+ entity TEXT NOT NULL
356
+ );
357
+ `);
358
+
359
+ // Migration: memory-spine provenance + temporal validity (D5/D8, 2026-07-30)
360
+ try {
361
+ this.db.prepare('SELECT origin_class FROM engrams LIMIT 0').get();
362
+ } catch {
363
+ this.db.exec(`
364
+ ALTER TABLE engrams ADD COLUMN origin_class TEXT;
365
+ ALTER TABLE engrams ADD COLUMN writer_session TEXT;
366
+ ALTER TABLE engrams ADD COLUMN recipe_id TEXT;
367
+ ALTER TABLE engrams ADD COLUMN valid_from TEXT;
368
+ ALTER TABLE engrams ADD COLUMN valid_to TEXT;
369
+ `);
370
+ }
371
+
372
+ // Migration: add conscious_state table for checkpointing
373
+ this.db.exec(`
374
+ CREATE TABLE IF NOT EXISTS conscious_state (
375
+ agent_id TEXT PRIMARY KEY,
376
+ last_write_id TEXT,
377
+ last_recall_context TEXT,
378
+ last_recall_ids TEXT NOT NULL DEFAULT '[]',
379
+ last_activity_at TEXT NOT NULL DEFAULT (datetime('now')),
380
+ write_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
381
+ recall_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
382
+ execution_state TEXT,
383
+ checkpoint_at TEXT,
384
+ last_consolidation_at TEXT,
385
+ last_mini_consolidation_at TEXT,
386
+ consolidation_cycle_count INTEGER NOT NULL DEFAULT 0,
387
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
388
+ )
389
+ `);
390
+
391
+ // Migration: add consolidation_cycle_count if missing (existing DBs)
392
+ try {
393
+ this.db.exec(`ALTER TABLE conscious_state ADD COLUMN consolidation_cycle_count INTEGER NOT NULL DEFAULT 0`);
394
+ } catch { /* column already exists */ }
395
+
396
+ // Migration: add memory_type column if missing
397
+ try {
398
+ this.db.prepare('SELECT memory_type FROM engrams LIMIT 0').get();
399
+ } catch {
400
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'unclassified'`);
401
+ }
402
+
403
+ // Migration (0.8 Cluster A): sequence column for story-time / chronology.
404
+ // Nullable — existing engrams stay NULL. Partial index keeps the cost low.
405
+ try {
406
+ this.db.prepare('SELECT sequence FROM engrams LIMIT 0').get();
407
+ } catch {
408
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN sequence INTEGER`);
409
+ this.db.exec(
410
+ `CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence
411
+ ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL`
412
+ );
413
+ }
414
+
415
+ // Migration (0.8 Cluster A): references_json column for typed cross-record
416
+ // links. Wired through HTTP in Cluster D — schema slot only here.
417
+ try {
418
+ this.db.prepare('SELECT references_json FROM engrams LIMIT 0').get();
419
+ } catch {
420
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN references_json TEXT`);
421
+ }
422
+ }
423
+
424
+ // --- Engram CRUD ---
425
+
426
+ createEngram(input: EngramCreate): Engram {
427
+ const now = new Date().toISOString();
428
+ const id = randomUUID();
429
+ const embeddingBlob = input.embedding
430
+ ? Buffer.from(new Float32Array(input.embedding).buffer)
431
+ : null;
432
+
433
+ this.db.prepare(`
434
+ INSERT INTO engrams (id, agent_id, concept, content, embedding, confidence, salience,
435
+ access_count, last_accessed, created_at, salience_features, reason_codes, stage, tags, episode_id,
436
+ ttl, memory_class, supersedes, task_status, task_priority, blocked_by, memory_type,
437
+ sequence, references_json, origin_class, writer_session, recipe_id, valid_from, valid_to)
438
+ VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
439
+ `).run(
440
+ id, input.agentId, input.concept, input.content, embeddingBlob,
441
+ input.confidence ?? 0.5,
442
+ input.salience ?? 0.5,
443
+ now, now,
444
+ JSON.stringify(input.salienceFeatures ?? DEFAULT_SALIENCE_FEATURES),
445
+ JSON.stringify(input.reasonCodes ?? []),
446
+ JSON.stringify(input.tags ?? []),
447
+ input.episodeId ?? null,
448
+ input.ttl ?? null,
449
+ input.memoryClass ?? 'working',
450
+ input.supersedes ?? null,
451
+ input.taskStatus ?? null,
452
+ input.taskPriority ?? null,
453
+ input.blockedBy ?? null,
454
+ input.memoryType ?? 'unclassified',
455
+ input.sequence ?? null,
456
+ input.references && input.references.length > 0
457
+ ? JSON.stringify(input.references) : null,
458
+ input.originClass ?? null,
459
+ input.writerSession ?? null,
460
+ input.recipeId ?? null,
461
+ input.validFrom ?? null,
462
+ input.validTo ?? null,
463
+ );
464
+
465
+ // Add to slim cache (skip if not yet populated — first slim fetch will load it)
466
+ if (this.slimCachePopulated) {
467
+ this.cacheAdd({
468
+ id,
469
+ agentId: input.agentId,
470
+ concept: input.concept,
471
+ embedding: input.embedding ?? null,
472
+ stage: 'active',
473
+ retracted: false,
474
+ });
475
+ }
476
+
477
+ return this.getEngram(id)!;
478
+ }
479
+
480
+ getEngram(id: string): Engram | null {
481
+ const row = this.db.prepare('SELECT * FROM engrams WHERE id = ?').get(id) as any;
482
+ return row ? this.rowToEngram(row) : null;
483
+ }
484
+
485
+ getEngramsByAgent(agentId: string, stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
486
+ let query = 'SELECT * FROM engrams WHERE agent_id = ?';
487
+ const params: any[] = [agentId];
488
+
489
+ if (stage) {
490
+ query += ' AND stage = ?';
491
+ params.push(stage);
492
+ }
493
+ if (!includeRetracted) {
494
+ query += ' AND retracted = 0';
495
+ }
496
+
497
+ return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
498
+ }
499
+
500
+ /**
501
+ * Slim variant that returns only (id, concept, embedding) — the minimum needed
502
+ * for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
503
+ * survival check). Avoids materializing the content blob, tag JSON, salience
504
+ * features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
505
+ *
506
+ * Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
507
+ * engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
508
+ * of that is row materialization of fields we don't read in the filter pass.
509
+ */
510
+ getEngramsByAgentSlim(
511
+ agentId: string,
512
+ stage?: EngramStage,
513
+ includeRetracted: boolean = false
514
+ ): Array<{ id: string; concept: string; embedding: number[] | null }> {
515
+ if (this.slimCacheEnabled) {
516
+ this.ensureSlimCachePopulated();
517
+ const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
518
+ for (const entry of this.slimCache.values()) {
519
+ if (entry.agentId !== agentId) continue;
520
+ if (stage && entry.stage !== stage) continue;
521
+ if (!includeRetracted && entry.retracted) continue;
522
+ result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
523
+ }
524
+ return result;
525
+ }
526
+ // Cache disabled fall back to direct SQL
527
+ let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
528
+ const params: any[] = [agentId];
529
+
530
+ if (stage) {
531
+ query += ' AND stage = ?';
532
+ params.push(stage);
533
+ }
534
+ if (!includeRetracted) {
535
+ query += ' AND retracted = 0';
536
+ }
537
+
538
+ return (this.db.prepare(query).all(...params) as any[]).map(r => ({
539
+ id: r.id as string,
540
+ concept: r.concept as string,
541
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
542
+ }));
543
+ }
544
+
545
+ /** Slim variant for multi-agent (workspace-scoped) pre-filter. */
546
+ getEngramsByAgentsSlim(
547
+ agentIds: string[],
548
+ stage?: EngramStage,
549
+ includeRetracted: boolean = false
550
+ ): Array<{ id: string; concept: string; embedding: number[] | null }> {
551
+ if (agentIds.length === 0) return [];
552
+ if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
553
+
554
+ if (this.slimCacheEnabled) {
555
+ this.ensureSlimCachePopulated();
556
+ const agentSet = new Set(agentIds);
557
+ const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
558
+ for (const entry of this.slimCache.values()) {
559
+ if (!agentSet.has(entry.agentId)) continue;
560
+ if (stage && entry.stage !== stage) continue;
561
+ if (!includeRetracted && entry.retracted) continue;
562
+ result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
563
+ }
564
+ return result;
565
+ }
566
+
567
+ const placeholders = agentIds.map(() => '?').join(',');
568
+ let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
569
+ const params: any[] = [...agentIds];
570
+
571
+ if (stage) {
572
+ query += ' AND stage = ?';
573
+ params.push(stage);
574
+ }
575
+ if (!includeRetracted) {
576
+ query += ' AND retracted = 0';
577
+ }
578
+
579
+ return (this.db.prepare(query).all(...params) as any[]).map(r => ({
580
+ id: r.id as string,
581
+ concept: r.concept as string,
582
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
583
+ }));
584
+ }
585
+
586
+ /**
587
+ * Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
588
+ * only the survivors that need deep scoring. Chunks IN-clause queries to stay
589
+ * under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
590
+ */
591
+ getEngramsByIds(ids: string[]): Engram[] {
592
+ if (ids.length === 0) return [];
593
+ const CHUNK = 800;
594
+ const result: Engram[] = [];
595
+ for (let i = 0; i < ids.length; i += CHUNK) {
596
+ const chunk = ids.slice(i, i + CHUNK);
597
+ const placeholders = chunk.map(() => '?').join(',');
598
+ const rows = this.db.prepare(
599
+ `SELECT * FROM engrams WHERE id IN (${placeholders})`
600
+ ).all(...chunk) as any[];
601
+ for (const r of rows) result.push(this.rowToEngram(r));
602
+ }
603
+ return result;
604
+ }
605
+
606
+ /**
607
+ * Get engrams across multiple agents (workspace-scoped recall).
608
+ * Used when workspace mode is enabled for hive memory sharing.
609
+ */
610
+ getEngramsByAgents(agentIds: string[], stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
611
+ if (agentIds.length === 0) return [];
612
+ if (agentIds.length === 1) return this.getEngramsByAgent(agentIds[0], stage, includeRetracted);
613
+
614
+ const placeholders = agentIds.map(() => '?').join(',');
615
+ let query = `SELECT * FROM engrams WHERE agent_id IN (${placeholders})`;
616
+ const params: any[] = [...agentIds];
617
+
618
+ if (stage) {
619
+ query += ' AND stage = ?';
620
+ params.push(stage);
621
+ }
622
+ if (!includeRetracted) {
623
+ query += ' AND retracted = 0';
624
+ }
625
+
626
+ return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
627
+ }
628
+
629
+ /**
630
+ * BM25 search across multiple agents (workspace-scoped).
631
+ */
632
+ searchBM25WithRankMultiAgent(agentIds: string[], query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
633
+ if (agentIds.length === 0) return [];
634
+ if (agentIds.length === 1) return this.searchBM25WithRank(agentIds[0], query, limit);
635
+
636
+ const sanitized = query
637
+ .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
638
+ .split(/\s+/)
639
+ .filter(w => w.length > 1)
640
+ .map(w => `"${w}"`)
641
+ .join(' OR ');
642
+
643
+ if (!sanitized) return [];
644
+
645
+ // CTE prefilter — see searchBM25WithRank for rationale (567× speedup verified).
646
+ try {
647
+ const placeholders = agentIds.map(() => '?').join(',');
648
+ const innerLimit = Math.max(limit * 5, 50);
649
+ const rows = this.db.prepare(`
650
+ WITH top_fts AS (
651
+ SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
652
+ )
653
+ SELECT e.*, top_fts.rank FROM top_fts
654
+ JOIN engrams e ON e.rowid = top_fts.rowid
655
+ WHERE e.agent_id IN (${placeholders}) AND e.retracted = 0
656
+ ORDER BY top_fts.rank
657
+ LIMIT ?
658
+ `).all(sanitized, innerLimit, ...agentIds, limit) as any[];
659
+
660
+ return rows.map(r => ({
661
+ engram: this.rowToEngram(r),
662
+ bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
663
+ }));
664
+ } catch {
665
+ return [];
666
+ }
667
+ }
668
+
669
+ /**
670
+ * Get all distinct agent IDs that share a workspace (requires coord_agents table).
671
+ * Returns just the queried agentId if coordination tables don't exist.
672
+ */
673
+ getWorkspaceAgentIds(agentId: string, workspace: string): string[] {
674
+ try {
675
+ // Return agent names (not UUIDs) — engrams.agent_id uses name strings
676
+ const rows = this.db.prepare(
677
+ `SELECT DISTINCT name FROM coord_agents WHERE workspace = ? AND status != 'dead'`
678
+ ).all(workspace) as Array<{ name: string }>;
679
+ const names = rows.map(r => r.name);
680
+ // Ensure the querying agent is always included
681
+ if (!names.includes(agentId)) names.push(agentId);
682
+ return names;
683
+ } catch {
684
+ // No coordination tables fall back to single agent
685
+ return [agentId];
686
+ }
687
+ }
688
+
689
+ /**
690
+ * Touch an engram: increment access count, update last_accessed, and
691
+ * nudge confidence upward. Each retrieval is weak evidence the memory
692
+ * is useful — bounded so only explicit feedback can push confidence
693
+ * above 0.85. Diminishing returns: first accesses matter most.
694
+ *
695
+ * Boost: +0.02 per access, scaled by 1/sqrt(accessCount+1), capped at 0.85.
696
+ */
697
+ touchEngram(id: string): void {
698
+ this.db.prepare(`
699
+ UPDATE engrams
700
+ SET access_count = access_count + 1,
701
+ last_accessed = ?,
702
+ confidence = MIN(0.85, confidence + 0.02 / (1.0 + sqrt(access_count)))
703
+ WHERE id = ?
704
+ `).run(new Date().toISOString(), id);
705
+ }
706
+
707
+ updateStage(id: string, stage: EngramStage): void {
708
+ this.db.prepare('UPDATE engrams SET stage = ? WHERE id = ?').run(stage, id);
709
+ this.cacheUpdateStage(id, stage);
710
+ }
711
+
712
+ /**
713
+ * Replace an engram's content. Used by the fade phase of consolidation
714
+ * (Paper 1: storage degradation) to coarsen un-recalled memories without
715
+ * deleting them — concept + tags stay intact for tag-based retrieval, but
716
+ * BM25 content surface area shrinks.
717
+ */
718
+ updateContent(id: string, content: string): void {
719
+ this.db.prepare('UPDATE engrams SET content = ? WHERE id = ?').run(content, id);
720
+ }
721
+
722
+ updateConfidence(id: string, confidence: number): void {
723
+ this.db.prepare('UPDATE engrams SET confidence = ? WHERE id = ?').run(
724
+ Math.max(0, Math.min(1, confidence)), id
725
+ );
726
+ }
727
+
728
+ updateEmbedding(id: string, embedding: number[], modelId?: string): void {
729
+ const blob = Buffer.from(new Float32Array(embedding).buffer);
730
+ if (modelId) {
731
+ this.db.prepare('UPDATE engrams SET embedding = ?, embedding_model = ? WHERE id = ?').run(blob, modelId, id);
732
+ } else {
733
+ this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
734
+ }
735
+ this.cacheUpdateEmbedding(id, embedding);
736
+ }
737
+
738
+ retractEngram(id: string, retractedBy: string | null): void {
739
+ this.db.prepare(`
740
+ UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
741
+ `).run(retractedBy, new Date().toISOString(), id);
742
+ this.cacheRetract(id);
743
+ }
744
+
745
+ deleteEngram(id: string): void {
746
+ this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
747
+ this.cacheRemove(id);
748
+ }
749
+
750
+ /**
751
+ * Time warp shift all timestamps backward by ms milliseconds.
752
+ * Used for testing time-dependent behavior (decay, forgetting).
753
+ * Returns count of records shifted.
754
+ */
755
+ timeWarp(agentId: string, ms: number): number {
756
+ let count = 0;
757
+ const shiftSec = Math.round(ms / 1000);
758
+ // Shift engram timestamps
759
+ const r1 = this.db.prepare(`
760
+ UPDATE engrams SET
761
+ created_at = datetime(created_at, '-${shiftSec} seconds'),
762
+ last_accessed = datetime(last_accessed, '-${shiftSec} seconds')
763
+ WHERE agent_id = ?
764
+ `).run(agentId);
765
+ count += r1.changes;
766
+ // Shift association timestamps
767
+ const r2 = this.db.prepare(`
768
+ UPDATE associations SET
769
+ created_at = datetime(created_at, '-${shiftSec} seconds'),
770
+ last_activated = datetime(last_activated, '-${shiftSec} seconds')
771
+ WHERE from_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
772
+ OR to_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
773
+ `).run(agentId, agentId);
774
+ count += r2.changes;
775
+ return count;
776
+ }
777
+
778
+ // --- Full-text search (BM25) ---
779
+
780
+ searchBM25(agentId: string, query: string, limit: number = 10): Engram[] {
781
+ return this.searchBM25WithRank(agentId, query, limit).map(r => r.engram);
782
+ }
783
+
784
+ /**
785
+ * Vector similarity top-K search. SQLite has no native vector index, so we
786
+ * iterate over the slim cache, compute cosine similarity in-process, and
787
+ * return the top-K nearest neighbors. PGlite uses pgvector + ivfflat for
788
+ * the same operation in O(log N).
789
+ *
790
+ * Returns engrams with their cosine distance (0 = identical, 2 = opposite)
791
+ * to match the PGlite contract. Activation engine consumes `1 - distance`
792
+ * as the cosine similarity score.
793
+ */
794
+ searchByVector(
795
+ agentId: string,
796
+ vec: number[],
797
+ limit: number = 10,
798
+ ): Array<{ engram: Engram; distance: number }> {
799
+ if (!vec || vec.length === 0) return [];
800
+ // Include both 'active' and 'fading' — faded engrams retain their embedding
801
+ // (Paper 1: storage degradation; only content is trimmed) so they should still
802
+ // participate in vector recall. Concept + tags + truncated content remain in BM25.
803
+ const activeSlim = this.getEngramsByAgentSlim(agentId, 'active', false);
804
+ const fadingSlim = this.getEngramsByAgentSlim(agentId, 'fading', false);
805
+ const slim = activeSlim.concat(fadingSlim);
806
+ const scored: Array<{ id: string; sim: number }> = [];
807
+ for (const e of slim) {
808
+ if (!e.embedding) continue;
809
+ const sim = cosineSimilarity(vec, e.embedding);
810
+ scored.push({ id: e.id, sim });
811
+ }
812
+ scored.sort((a, b) => b.sim - a.sim);
813
+ const topIds = scored.slice(0, limit).map(s => s.id);
814
+ if (topIds.length === 0) return [];
815
+ const engrams = this.getEngramsByIds(topIds);
816
+ const engramMap = new Map(engrams.map(e => [e.id, e]));
817
+ return scored
818
+ .slice(0, limit)
819
+ .map(s => ({ engram: engramMap.get(s.id), sim: s.sim }))
820
+ .filter((x): x is { engram: Engram; sim: number } => x.engram !== undefined)
821
+ .map(x => ({ engram: x.engram, distance: 1 - x.sim }));
822
+ }
823
+
824
+ /**
825
+ * BM25 search returning rank scores alongside engrams.
826
+ * FTS5 rank is negative (lower = better match).
827
+ * We normalize to 0-1 where higher = better.
828
+ */
829
+ searchBM25WithRank(agentId: string, query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
830
+ // Sanitize query for FTS5: quote each word to prevent column name interpretation
831
+ const sanitized = query
832
+ .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
833
+ .split(/\s+/)
834
+ .filter(w => w.length > 1)
835
+ .map(w => `"${w}"`)
836
+ .join(' OR ');
837
+
838
+ if (!sanitized) return [];
839
+
840
+ // CTE prefilter: force FTS5 to apply LIMIT before joining engrams.
841
+ //
842
+ // Why: the obvious query (JOIN engrams_fts ON rowid + WHERE MATCH + ORDER BY rank LIMIT N)
843
+ // makes SQLite's planner materialize ALL matching FTS rows joined with engrams
844
+ // before applying LIMIT. With wide OR queries on a 17K-engram index, that's
845
+ // thousands of row materializations including 1.5KB embedding blobs — measured
846
+ // at 3682ms for a 5-term OR query.
847
+ //
848
+ // The CTE forces FTS5 to LIMIT first (sub-ms), then join only the top-K rowids.
849
+ // Same query plan, 567× faster (3682ms → 6ms verified on 17K engrams).
850
+ //
851
+ // The inner LIMIT (limit * 5) over-fetches because the agent_id + retracted
852
+ // filter is applied AFTER the CTE. limit*5 gives enough headroom that filtered
853
+ // results still satisfy the outer LIMIT for typical workloads (single agent
854
+ // dominant, low retracted rate).
855
+ try {
856
+ const innerLimit = Math.max(limit * 5, 50);
857
+ const rows = this.db.prepare(`
858
+ WITH top_fts AS (
859
+ SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
860
+ )
861
+ SELECT e.*, top_fts.rank FROM top_fts
862
+ JOIN engrams e ON e.rowid = top_fts.rowid
863
+ WHERE e.agent_id = ? AND e.retracted = 0
864
+ ORDER BY top_fts.rank
865
+ LIMIT ?
866
+ `).all(sanitized, innerLimit, agentId, limit) as any[];
867
+
868
+ return rows.map(r => ({
869
+ engram: this.rowToEngram(r),
870
+ // Normalize: rank is negative, more negative = better match.
871
+ // |rank| / (1 + |rank|) gives 0-1 where higher = better.
872
+ bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
873
+ }));
874
+ } catch {
875
+ return [];
876
+ }
877
+ }
878
+
879
+ // --- Diagnostic search (deterministic, not cognitive) ---
880
+
881
+ search(query: SearchQuery): Engram[] {
882
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ?';
883
+ const params: any[] = [query.agentId];
884
+
885
+ if (query.text) {
886
+ sql += ' AND (content LIKE ? OR concept LIKE ?)';
887
+ params.push(`%${query.text}%`, `%${query.text}%`);
888
+ }
889
+ if (query.concept) {
890
+ sql += ' AND concept = ?';
891
+ params.push(query.concept);
892
+ }
893
+ if (query.stage) {
894
+ sql += ' AND stage = ?';
895
+ params.push(query.stage);
896
+ }
897
+ if (query.retracted !== undefined) {
898
+ sql += ' AND retracted = ?';
899
+ params.push(query.retracted ? 1 : 0);
900
+ }
901
+
902
+ // ── Tag filters (0.8 Cluster B) ──
903
+ // AND-set: merge legacy `tags` + new `tagsAll`. Both apply (intersection).
904
+ const allTags = [...(query.tags ?? []), ...(query.tagsAll ?? [])];
905
+ if (allTags.length > 0) {
906
+ for (const tag of allTags) {
907
+ sql += ' AND tags LIKE ?';
908
+ params.push(`%"${tag}"%`);
909
+ }
910
+ }
911
+ // OR-set: at least one must match.
912
+ if (query.tagsAny && query.tagsAny.length > 0) {
913
+ const ors = query.tagsAny.map(() => 'tags LIKE ?').join(' OR ');
914
+ sql += ` AND (${ors})`;
915
+ for (const tag of query.tagsAny) params.push(`%"${tag}"%`);
916
+ }
917
+ // NOT-set: none may match.
918
+ if (query.tagsNone && query.tagsNone.length > 0) {
919
+ const ors = query.tagsNone.map(() => 'tags LIKE ?').join(' OR ');
920
+ sql += ` AND NOT (${ors})`;
921
+ for (const tag of query.tagsNone) params.push(`%"${tag}"%`);
922
+ }
923
+
924
+ // ── Sort (0.8 Cluster B) ──
925
+ // Default preserves legacy behavior (last_accessed DESC). Map TS camelCase
926
+ // to SQL snake_case. `sequence` puts NULLs last via (col IS NULL) trick.
927
+ const sortCol = ({
928
+ createdAt: 'created_at',
929
+ sequence: 'sequence',
930
+ salience: 'salience',
931
+ confidence: 'confidence',
932
+ lastAccessed: 'last_accessed',
933
+ } as const)[query.sortBy ?? 'lastAccessed'];
934
+ const dir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
935
+ if (query.sortBy === 'sequence') {
936
+ // NULLs last regardless of direction so engrams without a story-time
937
+ // value don't shuffle into the middle of sequenced results.
938
+ sql += ` ORDER BY (sequence IS NULL), sequence ${dir}`;
939
+ } else {
940
+ sql += ` ORDER BY ${sortCol} ${dir}`;
941
+ }
942
+
943
+ sql += ` LIMIT ? OFFSET ?`;
944
+ params.push(query.limit ?? 50, query.offset ?? 0);
945
+
946
+ return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
947
+ }
948
+
949
+ /**
950
+ * Get the most recently created engram for an agent (for temporal adjacency edges).
951
+ */
952
+ getLatestEngram(agentId: string, excludeId?: string): Engram | null {
953
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0';
954
+ const params: any[] = [agentId];
955
+ if (excludeId) {
956
+ sql += ' AND id != ?';
957
+ params.push(excludeId);
958
+ }
959
+ sql += ' ORDER BY created_at DESC LIMIT 1';
960
+ const row = this.db.prepare(sql).get(...params) as any;
961
+ return row ? this.rowToEngram(row) : null;
962
+ }
963
+
964
+ // --- Task management ---
965
+
966
+ updateTaskStatus(id: string, status: TaskStatus): void {
967
+ this.db.prepare('UPDATE engrams SET task_status = ? WHERE id = ?').run(status, id);
968
+ }
969
+
970
+ updateTaskPriority(id: string, priority: TaskPriority): void {
971
+ this.db.prepare('UPDATE engrams SET task_priority = ? WHERE id = ?').run(priority, id);
972
+ }
973
+
974
+ updateBlockedBy(id: string, blockedBy: string | null): void {
975
+ this.db.prepare('UPDATE engrams SET blocked_by = ?, task_status = ? WHERE id = ?')
976
+ .run(blockedBy, blockedBy ? 'blocked' : 'open', id);
977
+ }
978
+
979
+ /**
980
+ * Get tasks for an agent, optionally filtered by status.
981
+ * Results ordered by priority (urgent > high > medium > low), then creation date.
982
+ */
983
+ getTasks(agentId: string, status?: TaskStatus): Engram[] {
984
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND task_status IS NOT NULL AND retracted = 0';
985
+ const params: any[] = [agentId];
986
+ if (status) {
987
+ sql += ' AND task_status = ?';
988
+ params.push(status);
989
+ }
990
+ sql += ` ORDER BY
991
+ CASE task_priority
992
+ WHEN 'urgent' THEN 0
993
+ WHEN 'high' THEN 1
994
+ WHEN 'medium' THEN 2
995
+ WHEN 'low' THEN 3
996
+ ELSE 4
997
+ END,
998
+ created_at DESC`;
999
+ return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
1000
+ }
1001
+
1002
+ /**
1003
+ * Get the next actionable task highest priority that's not blocked or done.
1004
+ */
1005
+ getNextTask(agentId: string): Engram | null {
1006
+ const row = this.db.prepare(`
1007
+ SELECT * FROM engrams
1008
+ WHERE agent_id = ? AND task_status IN ('open', 'in_progress') AND retracted = 0
1009
+ ORDER BY
1010
+ CASE task_status WHEN 'in_progress' THEN 0 ELSE 1 END,
1011
+ CASE task_priority
1012
+ WHEN 'urgent' THEN 0
1013
+ WHEN 'high' THEN 1
1014
+ WHEN 'medium' THEN 2
1015
+ WHEN 'low' THEN 3
1016
+ ELSE 4
1017
+ END,
1018
+ created_at ASC
1019
+ LIMIT 1
1020
+ `).get(agentId) as any;
1021
+ return row ? this.rowToEngram(row) : null;
1022
+ }
1023
+
1024
+ // --- Supersession ---
1025
+
1026
+ /**
1027
+ * Mark an engram as superseded by another.
1028
+ * The old memory stays in the DB (historical) but gets down-ranked in recall.
1029
+ */
1030
+ supersedeEngram(oldId: string, newId: string): void {
1031
+ this.db.prepare('UPDATE engrams SET superseded_by = ? WHERE id = ?').run(newId, oldId);
1032
+ this.db.prepare('UPDATE engrams SET supersedes = ? WHERE id = ?').run(oldId, newId);
1033
+ }
1034
+
1035
+ /**
1036
+ * Find the most recent active engram matching `concept` (case-insensitive,
1037
+ * trimmed) and all of `requiredTags`. Excludes superseded and retracted
1038
+ * engrams. Used by /memory/supersede Form B (0.8 Cluster D) and by
1039
+ * /memory/write's references[] resolution.
1040
+ *
1041
+ * Returns null if no match.
1042
+ */
1043
+ findActiveMatchByConcept(
1044
+ agentId: string,
1045
+ concept: string,
1046
+ requiredTags?: string[],
1047
+ ): Engram | null {
1048
+ let sql = `SELECT * FROM engrams
1049
+ WHERE agent_id = ?
1050
+ AND LOWER(TRIM(concept)) = LOWER(TRIM(?))
1051
+ AND stage = 'active'
1052
+ AND retracted = 0
1053
+ AND superseded_by IS NULL`;
1054
+ const params: any[] = [agentId, concept];
1055
+ if (requiredTags && requiredTags.length > 0) {
1056
+ for (const tag of requiredTags) {
1057
+ sql += ' AND tags LIKE ?';
1058
+ params.push(`%"${tag}"%`);
1059
+ }
1060
+ }
1061
+ sql += ' ORDER BY created_at DESC LIMIT 1';
1062
+ const row = this.db.prepare(sql).get(...params) as any;
1063
+ return row ? this.rowToEngram(row) : null;
1064
+ }
1065
+
1066
+ /**
1067
+ * Check if an engram has been superseded.
1068
+ */
1069
+ isSuperseded(id: string): boolean {
1070
+ const row = this.db.prepare('SELECT superseded_by FROM engrams WHERE id = ?').get(id) as any;
1071
+ return row?.superseded_by != null;
1072
+ }
1073
+
1074
+ updateMemoryClass(id: string, memoryClass: MemoryClass): void {
1075
+ this.db.prepare('UPDATE engrams SET memory_class = ? WHERE id = ?').run(memoryClass, id);
1076
+ }
1077
+
1078
+ updateTags(id: string, tags: string[]): void {
1079
+ this.db.prepare('UPDATE engrams SET tags = ? WHERE id = ?').run(JSON.stringify(tags), id);
1080
+ }
1081
+
1082
+ // --- Associations ---
1083
+
1084
+ upsertAssociation(
1085
+ fromId: string, toId: string, weight: number,
1086
+ type: AssociationType = 'hebbian', confidence: number = 0.5
1087
+ ): Association {
1088
+ const now = new Date().toISOString();
1089
+ const id = randomUUID();
1090
+
1091
+ this.db.prepare(`
1092
+ INSERT INTO associations (id, from_engram_id, to_engram_id, weight, confidence, type, activation_count, created_at, last_activated)
1093
+ VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)
1094
+ ON CONFLICT(from_engram_id, to_engram_id) DO UPDATE SET
1095
+ weight = ?, confidence = ?, last_activated = ?, activation_count = activation_count + 1
1096
+ `).run(id, fromId, toId, weight, confidence, type, now, now, weight, confidence, now);
1097
+
1098
+ return this.getAssociation(fromId, toId)!;
1099
+ }
1100
+
1101
+ getAssociation(fromId: string, toId: string): Association | null {
1102
+ const row = this.db.prepare(
1103
+ 'SELECT * FROM associations WHERE from_engram_id = ? AND to_engram_id = ?'
1104
+ ).get(fromId, toId) as any;
1105
+ return row ? this.rowToAssociation(row) : null;
1106
+ }
1107
+
1108
+ getAssociationsFor(engramId: string): Association[] {
1109
+ const rows = this.db.prepare(
1110
+ 'SELECT * FROM associations WHERE from_engram_id = ? OR to_engram_id = ?'
1111
+ ).all(engramId, engramId);
1112
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1113
+ }
1114
+
1115
+ /**
1116
+ * Aggregate association stats per engram — count of edges and sum of weights.
1117
+ *
1118
+ * Why: the activation scoring loop only uses `associations.length` (Hebbian gate)
1119
+ * and `sum of weights` (Hebbian mean + centrality). It doesn't read individual
1120
+ * association fields. Returning scalar stats avoids materializing thousands of
1121
+ * Association objects.
1122
+ *
1123
+ * Phase-breakdown spike (2026-05-08, post-0.7.10) showed
1124
+ * `getAssociationsForBatch` over ~300 survivors took 222ms (25% of recall floor).
1125
+ * Stats-only aggregate via a single GROUP BY drops this to ~20ms.
1126
+ *
1127
+ * Graph walk still needs full Association rows, but it operates on top-N
1128
+ * (~30 candidates) — its per-call `getAssociationsFor` is cheap.
1129
+ */
1130
+ getAssociationStatsForBatch(engramIds: string[]): Map<string, { count: number; sumWeight: number }> {
1131
+ const result = new Map<string, { count: number; sumWeight: number }>();
1132
+ if (engramIds.length === 0) return result;
1133
+
1134
+ const CHUNK = 400;
1135
+ for (let i = 0; i < engramIds.length; i += CHUNK) {
1136
+ const chunk = engramIds.slice(i, i + CHUNK);
1137
+ const placeholders = chunk.map(() => '?').join(',');
1138
+ // UNION ALL counts each association once per endpoint that's in the candidate
1139
+ // set same semantics as the existing getAssociationsForBatch which buckets
1140
+ // associations under both their from and to engram. Self-loops would be
1141
+ // double-counted, but they're rare in practice and the prior code handled them
1142
+ // identically.
1143
+ const rows = this.db.prepare(
1144
+ `SELECT id, SUM(cnt) AS count, SUM(sw) AS sum_weight FROM (
1145
+ SELECT from_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE from_engram_id IN (${placeholders})
1146
+ UNION ALL
1147
+ SELECT to_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE to_engram_id IN (${placeholders})
1148
+ )
1149
+ WHERE id IN (${placeholders})
1150
+ GROUP BY id`
1151
+ ).all(...chunk, ...chunk, ...chunk) as Array<{ id: string; count: number; sum_weight: number }>;
1152
+
1153
+ for (const r of rows) {
1154
+ result.set(r.id, { count: r.count, sumWeight: r.sum_weight });
1155
+ }
1156
+ }
1157
+ // Ensure every requested id has an entry (even zero-edge engrams)
1158
+ for (const id of engramIds) {
1159
+ if (!result.has(id)) result.set(id, { count: 0, sumWeight: 0 });
1160
+ }
1161
+ return result;
1162
+ }
1163
+
1164
+ /**
1165
+ * Batch variant of getAssociationsFor fetches associations for many engrams
1166
+ * in a single query, returning a Map keyed by engram id.
1167
+ *
1168
+ * Why: per-candidate `getAssociationsFor` calls inside the activation scoring
1169
+ * loop are an N+1. Measured at 1300ms for 10K candidates (sub-ms per call but
1170
+ * accumulating). One IN-clause query reduces this to ~50ms.
1171
+ */
1172
+ getAssociationsForBatch(engramIds: string[]): Map<string, Association[]> {
1173
+ const result = new Map<string, Association[]>();
1174
+ if (engramIds.length === 0) return result;
1175
+
1176
+ // SQLite's default SQLITE_LIMIT_VARIABLE_NUMBER is 999. Chunk to stay safely below.
1177
+ // We bind each id twice (from + to), so chunks of 400 use 800 placeholders.
1178
+ const CHUNK = 400;
1179
+ for (let i = 0; i < engramIds.length; i += CHUNK) {
1180
+ const chunk = engramIds.slice(i, i + CHUNK);
1181
+ const placeholders = chunk.map(() => '?').join(',');
1182
+ const rows = this.db.prepare(
1183
+ `SELECT * FROM associations
1184
+ WHERE from_engram_id IN (${placeholders}) OR to_engram_id IN (${placeholders})`
1185
+ ).all(...chunk, ...chunk) as any[];
1186
+ for (const r of rows) {
1187
+ const a = this.rowToAssociation(r);
1188
+ // Bucket by both endpoints — getAssociationsFor returns either-direction matches.
1189
+ const fromList = result.get(a.fromEngramId) ?? [];
1190
+ fromList.push(a);
1191
+ result.set(a.fromEngramId, fromList);
1192
+ if (a.toEngramId !== a.fromEngramId) {
1193
+ const toList = result.get(a.toEngramId) ?? [];
1194
+ toList.push(a);
1195
+ result.set(a.toEngramId, toList);
1196
+ }
1197
+ }
1198
+ }
1199
+ // Ensure every requested id has an entry (even if empty) so callers can
1200
+ // .get() without null-checking.
1201
+ for (const id of engramIds) {
1202
+ if (!result.has(id)) result.set(id, []);
1203
+ }
1204
+ return result;
1205
+ }
1206
+
1207
+ getOutgoingAssociations(engramId: string): Association[] {
1208
+ const rows = this.db.prepare(
1209
+ 'SELECT * FROM associations WHERE from_engram_id = ?'
1210
+ ).all(engramId);
1211
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1212
+ }
1213
+
1214
+ countAssociationsFor(engramId: string): number {
1215
+ const row = this.db.prepare(
1216
+ 'SELECT COUNT(*) as count FROM associations WHERE from_engram_id = ?'
1217
+ ).get(engramId) as any;
1218
+ return row.count;
1219
+ }
1220
+
1221
+ getWeakestAssociation(engramId: string): Association | null {
1222
+ const row = this.db.prepare(
1223
+ 'SELECT * FROM associations WHERE from_engram_id = ? ORDER BY weight ASC LIMIT 1'
1224
+ ).get(engramId) as any;
1225
+ return row ? this.rowToAssociation(row) : null;
1226
+ }
1227
+
1228
+ deleteAssociation(id: string): void {
1229
+ this.db.prepare('DELETE FROM associations WHERE id = ?').run(id);
1230
+ }
1231
+
1232
+ getAllAssociations(agentId: string): Association[] {
1233
+ const rows = this.db.prepare(`
1234
+ SELECT a.* FROM associations a
1235
+ JOIN engrams e ON a.from_engram_id = e.id
1236
+ WHERE e.agent_id = ?
1237
+ `).all(agentId);
1238
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1239
+ }
1240
+
1241
+ // --- Eviction ---
1242
+
1243
+ getEvictionCandidates(agentId: string, limit: number): Engram[] {
1244
+ // Lowest combined score: low salience + low access + low confidence + oldest
1245
+ const rows = this.db.prepare(`
1246
+ SELECT * FROM engrams
1247
+ WHERE agent_id = ? AND stage = 'active' AND retracted = 0
1248
+ ORDER BY (salience * 0.3 + confidence * 0.3 + (CAST(access_count AS REAL) / (access_count + 5)) * 0.2 +
1249
+ (1.0 / (1.0 + (julianday('now') - julianday(last_accessed)))) * 0.2) ASC
1250
+ LIMIT ?
1251
+ `).all(agentId, limit) as any[];
1252
+ return rows.map(r => this.rowToEngram(r));
1253
+ }
1254
+
1255
+ getActiveCount(agentId: string): number {
1256
+ const row = this.db.prepare(
1257
+ "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'active'"
1258
+ ).get(agentId) as any;
1259
+ return row.count;
1260
+ }
1261
+
1262
+ getStagingCount(agentId: string): number {
1263
+ const row = this.db.prepare(
1264
+ "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'staging'"
1265
+ ).get(agentId) as any;
1266
+ return row.count;
1267
+ }
1268
+
1269
+ // --- Staging buffer ---
1270
+
1271
+ getExpiredStaging(): Engram[] {
1272
+ const now = Date.now();
1273
+ const rows = this.db.prepare(`
1274
+ SELECT * FROM engrams WHERE stage = 'staging' AND ttl IS NOT NULL
1275
+ `).all() as any[];
1276
+
1277
+ return rows
1278
+ .map(r => this.rowToEngram(r))
1279
+ .filter(e => e.ttl && (e.createdAt.getTime() + e.ttl) < now);
1280
+ }
1281
+
1282
+ // --- Eval event logging ---
1283
+
1284
+ logActivationEvent(event: ActivationEvent): void {
1285
+ this.db.prepare(`
1286
+ INSERT INTO activation_events (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
1287
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1288
+ `).run(
1289
+ event.id, event.agentId, event.timestamp.toISOString(),
1290
+ event.context, event.resultsReturned, event.topScore,
1291
+ event.latencyMs, JSON.stringify(event.engramIds)
1292
+ );
1293
+ }
1294
+
1295
+ logStagingEvent(event: StagingEvent): void {
1296
+ this.db.prepare(`
1297
+ INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
1298
+ VALUES (?, ?, ?, ?, ?, ?)
1299
+ `).run(
1300
+ event.engramId, event.agentId, event.action,
1301
+ event.resonanceScore, event.timestamp.toISOString(), event.ageMs
1302
+ );
1303
+ }
1304
+
1305
+ logRetrievalFeedback(activationEventId: string | null, engramId: string, useful: boolean, context: string): void {
1306
+ this.db.prepare(`
1307
+ INSERT INTO retrieval_feedback (id, activation_event_id, engram_id, useful, context, timestamp)
1308
+ VALUES (?, ?, ?, ?, ?, ?)
1309
+ `).run(randomUUID(), activationEventId, engramId, useful ? 1 : 0, context, new Date().toISOString());
1310
+ }
1311
+
1312
+ // --- Eval metrics queries ---
1313
+
1314
+ getRetrievalPrecision(agentId: string, windowHours: number = 24): number {
1315
+ const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1316
+ const row = this.db.prepare(`
1317
+ SELECT
1318
+ COUNT(CASE WHEN useful = 1 THEN 1 END) as useful_count,
1319
+ COUNT(*) as total_count
1320
+ FROM retrieval_feedback rf
1321
+ LEFT JOIN activation_events ae ON rf.activation_event_id = ae.id
1322
+ JOIN engrams e ON rf.engram_id = e.id
1323
+ WHERE e.agent_id = ? AND rf.timestamp > ?
1324
+ `).get(agentId, since) as any;
1325
+
1326
+ return row.total_count > 0 ? row.useful_count / row.total_count : 0;
1327
+ }
1328
+
1329
+ getStagingMetrics(agentId: string): { promoted: number; discarded: number; expired: number } {
1330
+ const row = this.db.prepare(`
1331
+ SELECT
1332
+ COUNT(CASE WHEN action = 'promoted' THEN 1 END) as promoted,
1333
+ COUNT(CASE WHEN action = 'discarded' THEN 1 END) as discarded,
1334
+ COUNT(CASE WHEN action = 'expired' THEN 1 END) as expired
1335
+ FROM staging_events WHERE agent_id = ?
1336
+ `).get(agentId) as any;
1337
+ return { promoted: row.promoted, discarded: row.discarded, expired: row.expired };
1338
+ }
1339
+
1340
+ getActivationStats(agentId: string, windowHours: number = 24): {
1341
+ count: number; avgLatencyMs: number; p95LatencyMs: number;
1342
+ } {
1343
+ const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1344
+ const rows = this.db.prepare(`
1345
+ SELECT latency_ms FROM activation_events
1346
+ WHERE agent_id = ? AND timestamp > ?
1347
+ ORDER BY latency_ms ASC
1348
+ `).all(agentId, since) as { latency_ms: number }[];
1349
+
1350
+ if (rows.length === 0) return { count: 0, avgLatencyMs: 0, p95LatencyMs: 0 };
1351
+
1352
+ const total = rows.reduce((s, r) => s + r.latency_ms, 0);
1353
+ const p95Index = Math.min(Math.floor(rows.length * 0.95), rows.length - 1);
1354
+ return {
1355
+ count: rows.length,
1356
+ avgLatencyMs: total / rows.length,
1357
+ p95LatencyMs: rows[p95Index].latency_ms,
1358
+ };
1359
+ }
1360
+
1361
+ getConsolidatedCount(agentId: string): number {
1362
+ const row = this.db.prepare(
1363
+ `SELECT COUNT(*) as cnt FROM engrams WHERE agent_id = ? AND stage = 'consolidated'`
1364
+ ).get(agentId) as any;
1365
+ return row.cnt;
1366
+ }
1367
+
1368
+ // --- Helpers ---
1369
+
1370
+ private rowToEngram(row: any): Engram {
1371
+ return {
1372
+ id: row.id,
1373
+ agentId: row.agent_id,
1374
+ concept: row.concept,
1375
+ content: row.content,
1376
+ embedding: row.embedding
1377
+ ? Array.from(bufferToFloat32Array(row.embedding))
1378
+ : null,
1379
+ confidence: row.confidence,
1380
+ salience: row.salience,
1381
+ accessCount: row.access_count,
1382
+ lastAccessed: new Date(row.last_accessed),
1383
+ createdAt: new Date(row.created_at),
1384
+ salienceFeatures: JSON.parse(row.salience_features || '{}'),
1385
+ reasonCodes: JSON.parse(row.reason_codes || '[]'),
1386
+ stage: row.stage as EngramStage,
1387
+ ttl: row.ttl,
1388
+ retracted: !!row.retracted,
1389
+ retractedBy: row.retracted_by,
1390
+ retractedAt: row.retracted_at ? new Date(row.retracted_at) : null,
1391
+ tags: JSON.parse(row.tags),
1392
+ episodeId: row.episode_id ?? null,
1393
+ memoryClass: (row.memory_class ?? 'working') as MemoryClass,
1394
+ memoryType: (row.memory_type ?? 'unclassified') as MemoryType,
1395
+ supersededBy: row.superseded_by ?? null,
1396
+ supersedes: row.supersedes ?? null,
1397
+ taskStatus: row.task_status ?? null,
1398
+ taskPriority: row.task_priority ?? null,
1399
+ blockedBy: row.blocked_by ?? null,
1400
+ sequence: row.sequence == null ? null : Number(row.sequence),
1401
+ references: row.references_json ? JSON.parse(row.references_json) : null,
1402
+ originClass: row.origin_class ?? null,
1403
+ writerSession: row.writer_session ?? null,
1404
+ recipeId: row.recipe_id ?? null,
1405
+ validFrom: row.valid_from ?? null,
1406
+ validTo: row.valid_to ?? null,
1407
+ };
1408
+ }
1409
+
1410
+ private rowToAssociation(row: any): Association {
1411
+ return {
1412
+ id: row.id,
1413
+ fromEngramId: row.from_engram_id,
1414
+ toEngramId: row.to_engram_id,
1415
+ weight: row.weight,
1416
+ confidence: row.confidence ?? 0.5,
1417
+ type: row.type as AssociationType,
1418
+ activationCount: row.activation_count ?? 0,
1419
+ createdAt: new Date(row.created_at),
1420
+ lastActivated: new Date(row.last_activated),
1421
+ };
1422
+ }
1423
+
1424
+ // --- Episodes ---
1425
+
1426
+ createEpisode(input: { agentId: string; label: string; embedding?: number[] }): Episode {
1427
+ const now = new Date().toISOString();
1428
+ const id = randomUUID();
1429
+ const embeddingBlob = input.embedding
1430
+ ? Buffer.from(new Float32Array(input.embedding).buffer)
1431
+ : null;
1432
+
1433
+ this.db.prepare(`
1434
+ INSERT INTO episodes (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
1435
+ VALUES (?, ?, ?, ?, 0, ?, ?, ?)
1436
+ `).run(id, input.agentId, input.label, embeddingBlob, now, now, now);
1437
+
1438
+ return this.getEpisode(id)!;
1439
+ }
1440
+
1441
+ getEpisode(id: string): Episode | null {
1442
+ const row = this.db.prepare('SELECT * FROM episodes WHERE id = ?').get(id) as any;
1443
+ return row ? this.rowToEpisode(row) : null;
1444
+ }
1445
+
1446
+ getEpisodesByAgent(agentId: string): Episode[] {
1447
+ const rows = this.db.prepare(
1448
+ 'SELECT * FROM episodes WHERE agent_id = ? ORDER BY end_time DESC'
1449
+ ).all(agentId) as any[];
1450
+ return rows.map(r => this.rowToEpisode(r));
1451
+ }
1452
+
1453
+ getActiveEpisode(agentId: string, windowMs: number = 3600_000): Episode | null {
1454
+ // Find most recent episode that ended within the time window
1455
+ const cutoff = new Date(Date.now() - windowMs).toISOString();
1456
+ const row = this.db.prepare(`
1457
+ SELECT * FROM episodes WHERE agent_id = ? AND end_time > ?
1458
+ ORDER BY end_time DESC LIMIT 1
1459
+ `).get(agentId, cutoff) as any;
1460
+ return row ? this.rowToEpisode(row) : null;
1461
+ }
1462
+
1463
+ addEngramToEpisode(engramId: string, episodeId: string): void {
1464
+ this.db.prepare('UPDATE engrams SET episode_id = ? WHERE id = ?').run(episodeId, engramId);
1465
+ this.db.prepare(`
1466
+ UPDATE episodes SET
1467
+ engram_count = engram_count + 1,
1468
+ end_time = MAX(end_time, ?)
1469
+ WHERE id = ?
1470
+ `).run(new Date().toISOString(), episodeId);
1471
+ }
1472
+
1473
+ getEngramsByEpisode(episodeId: string): Engram[] {
1474
+ const rows = this.db.prepare(
1475
+ 'SELECT * FROM engrams WHERE episode_id = ? AND retracted = 0 ORDER BY created_at ASC'
1476
+ ).all(episodeId) as any[];
1477
+ return rows.map(r => this.rowToEngram(r));
1478
+ }
1479
+
1480
+ updateEpisodeEmbedding(id: string, embedding: number[]): void {
1481
+ const blob = Buffer.from(new Float32Array(embedding).buffer);
1482
+ this.db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(blob, id);
1483
+ }
1484
+
1485
+ getEpisodeCount(agentId: string): number {
1486
+ const row = this.db.prepare(
1487
+ 'SELECT COUNT(*) as cnt FROM episodes WHERE agent_id = ?'
1488
+ ).get(agentId) as any;
1489
+ return row.cnt;
1490
+ }
1491
+
1492
+ private rowToEpisode(row: any): Episode {
1493
+ return {
1494
+ id: row.id,
1495
+ agentId: row.agent_id,
1496
+ label: row.label,
1497
+ embedding: row.embedding
1498
+ ? Array.from(bufferToFloat32Array(row.embedding))
1499
+ : null,
1500
+ engramCount: row.engram_count,
1501
+ startTime: new Date(row.start_time),
1502
+ endTime: new Date(row.end_time),
1503
+ createdAt: new Date(row.created_at),
1504
+ };
1505
+ }
1506
+
1507
+ /**
1508
+ * Find engrams whose tags contain any of the given tag values.
1509
+ * Used for entity-bridge retrieval: given entity tags from top results,
1510
+ * find other engrams mentioning the same entities.
1511
+ */
1512
+ findEngramsByTags(agentId: string, tags: string[], excludeIds?: Set<string>): Engram[] {
1513
+ if (tags.length === 0) return [];
1514
+
1515
+ // Build OR conditions for tag matching
1516
+ const conditions = tags.map(() => 'tags LIKE ?').join(' OR ');
1517
+ const params: any[] = [agentId, ...tags.map(t => `%"${t}"%`)];
1518
+
1519
+ let sql = `SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0 AND (${conditions})`;
1520
+ const rows = this.db.prepare(sql).all(...params) as any[];
1521
+
1522
+ const results = rows.map(r => this.rowToEngram(r));
1523
+ if (excludeIds) {
1524
+ return results.filter(e => !excludeIds.has(e.id));
1525
+ }
1526
+ return results;
1527
+ }
1528
+
1529
+ // --- Checkpointing ---
1530
+
1531
+ updateAutoCheckpointWrite(agentId: string, engramId: string): void {
1532
+ const now = new Date().toISOString();
1533
+ this.db.prepare(`
1534
+ INSERT INTO conscious_state (agent_id, last_write_id, last_activity_at, write_count_since_consolidation, updated_at)
1535
+ VALUES (?, ?, ?, 1, ?)
1536
+ ON CONFLICT(agent_id) DO UPDATE SET
1537
+ last_write_id = excluded.last_write_id,
1538
+ last_activity_at = excluded.last_activity_at,
1539
+ write_count_since_consolidation = write_count_since_consolidation + 1,
1540
+ updated_at = excluded.updated_at
1541
+ `).run(agentId, engramId, now, now);
1542
+ }
1543
+
1544
+ updateAutoCheckpointRecall(agentId: string, context: string, engramIds: string[]): void {
1545
+ const now = new Date().toISOString();
1546
+ this.db.prepare(`
1547
+ INSERT INTO conscious_state (agent_id, last_recall_context, last_recall_ids, last_activity_at, recall_count_since_consolidation, updated_at)
1548
+ VALUES (?, ?, ?, ?, 1, ?)
1549
+ ON CONFLICT(agent_id) DO UPDATE SET
1550
+ last_recall_context = excluded.last_recall_context,
1551
+ last_recall_ids = excluded.last_recall_ids,
1552
+ last_activity_at = excluded.last_activity_at,
1553
+ recall_count_since_consolidation = recall_count_since_consolidation + 1,
1554
+ updated_at = excluded.updated_at
1555
+ `).run(agentId, context, JSON.stringify(engramIds), now, now);
1556
+ }
1557
+
1558
+ touchActivity(agentId: string): void {
1559
+ const now = new Date().toISOString();
1560
+ this.db.prepare(`
1561
+ INSERT INTO conscious_state (agent_id, last_activity_at, updated_at)
1562
+ VALUES (?, ?, ?)
1563
+ ON CONFLICT(agent_id) DO UPDATE SET
1564
+ last_activity_at = excluded.last_activity_at,
1565
+ updated_at = excluded.updated_at
1566
+ `).run(agentId, now, now);
1567
+ }
1568
+
1569
+ saveCheckpoint(agentId: string, state: ConsciousState): void {
1570
+ const now = new Date().toISOString();
1571
+ this.db.prepare(`
1572
+ INSERT INTO conscious_state (agent_id, execution_state, checkpoint_at, last_activity_at, updated_at)
1573
+ VALUES (?, ?, ?, ?, ?)
1574
+ ON CONFLICT(agent_id) DO UPDATE SET
1575
+ execution_state = excluded.execution_state,
1576
+ checkpoint_at = excluded.checkpoint_at,
1577
+ last_activity_at = excluded.last_activity_at,
1578
+ updated_at = excluded.updated_at
1579
+ `).run(agentId, JSON.stringify(state), now, now, now);
1580
+ }
1581
+
1582
+ /** Record entity mentions for an engram (D9 inverted index; idempotent). */
1583
+ recordEntityMentions(engramId: string, agentId: string, entities: string[]): void {
1584
+ if (!entities || entities.length === 0) return;
1585
+ const stmt = this.db.prepare('INSERT OR IGNORE INTO entity_mentions (entity, engram_id, agent_id) VALUES (?, ?, ?)');
1586
+ for (const entity of entities) stmt.run(entity, engramId, agentId);
1587
+ }
1588
+
1589
+ /** Indexed entities whose normalized key:value or a registered alias — contains the term
1590
+ * (D11 query-side lookup). Alias matches resolve to their target entity. */
1591
+ searchEntities(term: string, limit: number = 8): string[] {
1592
+ const t = term.toLowerCase().replace(/[%_]/g, '');
1593
+ if (!t) return [];
1594
+ const rows = this.db.prepare(`
1595
+ SELECT DISTINCT entity FROM (
1596
+ SELECT entity FROM entity_mentions WHERE entity LIKE ?
1597
+ UNION SELECT entity FROM entity_aliases WHERE alias LIKE ?
1598
+ ) LIMIT ?`).all(`%${t}%`, `%${t}%`, limit) as Array<{ entity: string }>;
1599
+ return rows.map(r => r.entity);
1600
+ }
1601
+
1602
+ /** Engram ids mentioning an entity (alias-resolved), optionally agent-scoped. */
1603
+ getEngramIdsByEntity(entity: string, agentId?: string): string[] {
1604
+ const resolved = (this.db.prepare('SELECT entity FROM entity_aliases WHERE alias = ?')
1605
+ .get(entity.toLowerCase()) as { entity: string } | undefined)?.entity ?? entity.toLowerCase();
1606
+ const rows = agentId
1607
+ ? this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ? AND agent_id = ?').all(resolved, agentId)
1608
+ : this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ?').all(resolved);
1609
+ return (rows as Array<{ engram_id: string }>).map(r => r.engram_id);
1610
+ }
1611
+
1612
+ /** Distinct agent ids present in this store (D3 whoami; includes all stages). */
1613
+ listAgentIds(): string[] {
1614
+ const rows = this.db.prepare('SELECT DISTINCT agent_id FROM engrams ORDER BY agent_id').all() as Array<{ agent_id: string }>;
1615
+ return rows.map(r => r.agent_id);
1616
+ }
1617
+
1618
+ getCheckpoint(agentId: string): CheckpointRow | null {
1619
+ const row = this.db.prepare('SELECT * FROM conscious_state WHERE agent_id = ?').get(agentId) as any;
1620
+ if (!row) return null;
1621
+
1622
+ return {
1623
+ agentId: row.agent_id,
1624
+ auto: {
1625
+ lastWriteId: row.last_write_id ?? null,
1626
+ lastRecallContext: row.last_recall_context ?? null,
1627
+ lastRecallIds: JSON.parse(row.last_recall_ids || '[]'),
1628
+ lastActivityAt: new Date(row.last_activity_at),
1629
+ writeCountSinceConsolidation: row.write_count_since_consolidation,
1630
+ recallCountSinceConsolidation: row.recall_count_since_consolidation,
1631
+ },
1632
+ executionState: row.execution_state ? JSON.parse(row.execution_state) : null,
1633
+ checkpointAt: row.checkpoint_at ? new Date(row.checkpoint_at) : null,
1634
+ lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1635
+ lastMiniConsolidationAt: row.last_mini_consolidation_at ? new Date(row.last_mini_consolidation_at) : null,
1636
+ updatedAt: new Date(row.updated_at),
1637
+ };
1638
+ }
1639
+
1640
+ markConsolidation(agentId: string, mini: boolean): void {
1641
+ const now = new Date().toISOString();
1642
+ if (mini) {
1643
+ this.db.prepare(`
1644
+ UPDATE conscious_state SET last_mini_consolidation_at = ?, updated_at = ? WHERE agent_id = ?
1645
+ `).run(now, now, agentId);
1646
+ } else {
1647
+ this.db.prepare(`
1648
+ UPDATE conscious_state SET
1649
+ last_consolidation_at = ?,
1650
+ last_mini_consolidation_at = ?,
1651
+ write_count_since_consolidation = 0,
1652
+ recall_count_since_consolidation = 0,
1653
+ consolidation_cycle_count = consolidation_cycle_count + 1,
1654
+ updated_at = ?
1655
+ WHERE agent_id = ?
1656
+ `).run(now, now, now, agentId);
1657
+ }
1658
+ }
1659
+
1660
+ getActiveAgents(): Array<{ agentId: string; lastActivityAt: Date; writeCount: number; recallCount: number; lastConsolidationAt: Date | null }> {
1661
+ const rows = this.db.prepare('SELECT * FROM conscious_state').all() as any[];
1662
+ return rows.map(row => ({
1663
+ agentId: row.agent_id,
1664
+ lastActivityAt: new Date(row.last_activity_at),
1665
+ writeCount: row.write_count_since_consolidation,
1666
+ recallCount: row.recall_count_since_consolidation,
1667
+ lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1668
+ }));
1669
+ }
1670
+
1671
+ getConsolidationCycleCount(agentId: string): number {
1672
+ const row = this.db.prepare(
1673
+ 'SELECT consolidation_cycle_count FROM conscious_state WHERE agent_id = ?',
1674
+ ).get(agentId) as { consolidation_cycle_count: number } | undefined;
1675
+ return row?.consolidation_cycle_count ?? 0;
1676
+ }
1677
+
1678
+ close(): void {
1679
+ this.stopWalCheckpointTimer();
1680
+ this.walCheckpoint();
1681
+ this.db.close();
1682
+ }
1683
+
1684
+ /**
1685
+ * Run `fn` in a SQL transaction. All writes succeed atomically or roll back
1686
+ * on throw. Synchronous (matches better-sqlite3 semantics).
1687
+ *
1688
+ * Used by 0.8 Cluster D's `/memory/supersede` Form B to wrap find-match +
1689
+ * performWrite + supersede + causal-edge + confidence-decay so callers
1690
+ * never observe an intermediate state where new engram exists but old
1691
+ * isn't yet superseded.
1692
+ */
1693
+ transaction<T>(fn: () => T): T {
1694
+ return this.db.transaction(fn)();
1695
+ }
1696
+
1697
+ /**
1698
+ * Async-aware transaction wrapper. `fn` may await — we hold the SQLite
1699
+ * lock across awaits by issuing raw BEGIN/COMMIT/ROLLBACK statements rather
1700
+ * than using better-sqlite3's sync `db.transaction()`.
1701
+ *
1702
+ * Used by Form B (atomic write + supersede) where the body calls
1703
+ * async store methods through the IEngramStore contract.
1704
+ */
1705
+ async withTransaction<T>(fn: () => Promise<T>): Promise<T> {
1706
+ this.db.exec('BEGIN');
1707
+ try {
1708
+ const result = await fn();
1709
+ this.db.exec('COMMIT');
1710
+ return result;
1711
+ } catch (err) {
1712
+ try { this.db.exec('ROLLBACK'); } catch { /* best-effort */ }
1713
+ throw err;
1714
+ }
1715
+ }
1716
+
1717
+ // ============================================================
1718
+ // 0.8 Cluster C — materialized-view + atomic-counter primitives
1719
+ // ============================================================
1720
+
1721
+ /**
1722
+ * For each distinct value of the tag whose key prefix is `tagKeyPrefix`
1723
+ * (e.g. `"character="`), return the most-recent active engram. Used by
1724
+ * NovelForge's "latest emotional state per character", "latest motif
1725
+ * phase per motif", "recent chapter summaries".
1726
+ *
1727
+ * `sortBy` defaults to `createdAt`. With `"sequence"`, engrams without a
1728
+ * sequence are excluded (they have no story-time anchor).
1729
+ */
1730
+ getLatestByTag(opts: {
1731
+ agentId: string;
1732
+ tagKeyPrefix: string; // e.g. "character=", "motif=", "chapter="
1733
+ scopeTagsAll?: string[]; // optional narrowing (e.g. ["topic=motif-use"])
1734
+ retracted?: boolean;
1735
+ sortBy?: 'createdAt' | 'sequence';
1736
+ limit?: number;
1737
+ }): Engram[] {
1738
+ let sql = `SELECT * FROM engrams
1739
+ WHERE agent_id = ?
1740
+ AND retracted = ?
1741
+ AND stage = 'active'
1742
+ AND tags LIKE ?`;
1743
+ const params: any[] = [
1744
+ opts.agentId,
1745
+ opts.retracted ? 1 : 0,
1746
+ `%"${opts.tagKeyPrefix}%`,
1747
+ ];
1748
+ if (opts.scopeTagsAll && opts.scopeTagsAll.length > 0) {
1749
+ for (const t of opts.scopeTagsAll) {
1750
+ sql += ' AND tags LIKE ?';
1751
+ params.push(`%"${t}"%`);
1752
+ }
1753
+ }
1754
+ if (opts.sortBy === 'sequence') {
1755
+ sql += ' AND sequence IS NOT NULL';
1756
+ }
1757
+ sql += ' ORDER BY ' + (opts.sortBy === 'sequence' ? 'sequence DESC, created_at DESC' : 'created_at DESC');
1758
+
1759
+ const rows = this.db.prepare(sql).all(...params) as any[];
1760
+ const engrams = rows.map(r => this.rowToEngram(r));
1761
+
1762
+ // Group by extracted tag value, take first (already sorted newest first).
1763
+ const seen = new Map<string, Engram>();
1764
+ for (const e of engrams) {
1765
+ const value = this.extractTagValue(e.tags, opts.tagKeyPrefix);
1766
+ if (value == null) continue;
1767
+ if (!seen.has(value)) seen.set(value, e);
1768
+ }
1769
+ const out = Array.from(seen.values());
1770
+ return opts.limit ? out.slice(0, opts.limit) : out;
1771
+ }
1772
+
1773
+ /**
1774
+ * Filter engrams by tag-set operators and sort by numeric value extracted
1775
+ * from a tag prefix (e.g. `"weight="` → numeric value of `weight=8`).
1776
+ * Used by NovelForge's "top 40 active promises by weight".
1777
+ *
1778
+ * Missing or unparseable numeric values sort last.
1779
+ */
1780
+ getTopBy(opts: {
1781
+ agentId: string;
1782
+ sortField: string; // tag prefix, e.g. "weight="
1783
+ order: 'asc' | 'desc';
1784
+ filterTagsAll?: string[];
1785
+ filterTagsAny?: string[];
1786
+ filterTagsNone?: string[];
1787
+ limit?: number;
1788
+ retracted?: boolean;
1789
+ }): Engram[] {
1790
+ // Build the base query inline (mirrors search() but adds the topBy sort).
1791
+ let sql = `SELECT * FROM engrams
1792
+ WHERE agent_id = ?
1793
+ AND retracted = ?
1794
+ AND stage = 'active'
1795
+ AND tags LIKE ?`;
1796
+ const params: any[] = [
1797
+ opts.agentId,
1798
+ opts.retracted ? 1 : 0,
1799
+ `%"${opts.sortField}%`, // must have a tag with this prefix
1800
+ ];
1801
+
1802
+ if (opts.filterTagsAll && opts.filterTagsAll.length > 0) {
1803
+ for (const tag of opts.filterTagsAll) {
1804
+ sql += ' AND tags LIKE ?';
1805
+ params.push(`%"${tag}"%`);
1806
+ }
1807
+ }
1808
+ if (opts.filterTagsAny && opts.filterTagsAny.length > 0) {
1809
+ const ors = opts.filterTagsAny.map(() => 'tags LIKE ?').join(' OR ');
1810
+ sql += ` AND (${ors})`;
1811
+ for (const tag of opts.filterTagsAny) params.push(`%"${tag}"%`);
1812
+ }
1813
+ if (opts.filterTagsNone && opts.filterTagsNone.length > 0) {
1814
+ const ors = opts.filterTagsNone.map(() => 'tags LIKE ?').join(' OR ');
1815
+ sql += ` AND NOT (${ors})`;
1816
+ for (const tag of opts.filterTagsNone) params.push(`%"${tag}"%`);
1817
+ }
1818
+
1819
+ const rows = this.db.prepare(sql).all(...params) as any[];
1820
+ const engrams = rows.map(r => this.rowToEngram(r));
1821
+
1822
+ // Extract numeric value from the sortField tag; missing or NaN sorts last.
1823
+ const valued = engrams.map(e => {
1824
+ const raw = this.extractTagValue(e.tags, opts.sortField);
1825
+ const n = raw == null ? NaN : Number(raw);
1826
+ return { e, n };
1827
+ });
1828
+ valued.sort((a, b) => {
1829
+ const aNaN = Number.isNaN(a.n);
1830
+ const bNaN = Number.isNaN(b.n);
1831
+ if (aNaN && bNaN) return 0;
1832
+ if (aNaN) return 1; // NaN last
1833
+ if (bNaN) return -1;
1834
+ return opts.order === 'asc' ? a.n - b.n : b.n - a.n;
1835
+ });
1836
+ const sorted = valued.map(v => v.e);
1837
+ return opts.limit ? sorted.slice(0, opts.limit) : sorted;
1838
+ }
1839
+
1840
+ /**
1841
+ * Compute the effective state of an engram from referenced events.
1842
+ *
1843
+ * - 'superseded' if engram.supersededBy is set
1844
+ * - else: scan engrams whose references_json names this engram with a
1845
+ * terminal relation type (resolves / subverts / abandons). Take the
1846
+ * latest by createdAt. effectiveState = that type.
1847
+ * - else: 'active'
1848
+ *
1849
+ * Returns `null` if target engram not found.
1850
+ */
1851
+ resolveEffectiveState(targetEngramId: string): {
1852
+ engram: Engram;
1853
+ effectiveState: 'active' | 'resolved' | 'subverted' | 'abandoned' | 'superseded';
1854
+ resolvingEvents: Array<{ id: string; type: string; createdAt: string }>;
1855
+ } | null {
1856
+ const target = this.getEngram(targetEngramId);
1857
+ if (!target) return null;
1858
+
1859
+ if (target.supersededBy) {
1860
+ return { engram: target, effectiveState: 'superseded', resolvingEvents: [] };
1861
+ }
1862
+
1863
+ // Find candidate referencing engrams. LIKE-filter first to narrow,
1864
+ // then JSON.parse each candidate's references_json to verify the
1865
+ // matchEngramId and type. Cheap enough at NovelForge's scale.
1866
+ const candidates = this.db.prepare(
1867
+ `SELECT id, references_json, created_at FROM engrams
1868
+ WHERE agent_id = ?
1869
+ AND retracted = 0
1870
+ AND stage = 'active'
1871
+ AND references_json IS NOT NULL
1872
+ AND references_json LIKE ?`
1873
+ ).all(target.agentId, `%"${targetEngramId}"%`) as Array<{
1874
+ id: string;
1875
+ references_json: string;
1876
+ created_at: string;
1877
+ }>;
1878
+
1879
+ const terminalTypes = new Set(['resolves', 'subverts', 'abandons']);
1880
+ const resolvingEvents: Array<{ id: string; type: string; createdAt: string }> = [];
1881
+ for (const c of candidates) {
1882
+ try {
1883
+ const refs = JSON.parse(c.references_json) as Array<{ type: string; matchEngramId?: string }>;
1884
+ for (const ref of refs) {
1885
+ if (ref.matchEngramId === targetEngramId && terminalTypes.has(ref.type)) {
1886
+ resolvingEvents.push({ id: c.id, type: ref.type, createdAt: c.created_at });
1887
+ }
1888
+ }
1889
+ } catch { /* malformed references_json — skip */ }
1890
+ }
1891
+
1892
+ if (resolvingEvents.length === 0) {
1893
+ return { engram: target, effectiveState: 'active', resolvingEvents: [] };
1894
+ }
1895
+
1896
+ // Latest event wins.
1897
+ resolvingEvents.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
1898
+ const latestType = resolvingEvents[0]!.type as 'resolves' | 'subverts' | 'abandons';
1899
+ const effectiveState = (
1900
+ latestType === 'resolves' ? 'resolved'
1901
+ : latestType === 'subverts' ? 'subverted'
1902
+ : 'abandoned'
1903
+ );
1904
+ return { engram: target, effectiveState, resolvingEvents };
1905
+ }
1906
+
1907
+ /**
1908
+ * Atomically allocate the next sequence number for an agent.
1909
+ *
1910
+ * Uses BEGIN IMMEDIATE to serialize concurrent callers — better-sqlite3's
1911
+ * transaction() acquires a RESERVED lock which prevents other writers
1912
+ * from racing. Returns MAX(sequence)+1 for the agent, or 1 if no engrams
1913
+ * have a sequence yet.
1914
+ */
1915
+ allocateNextSequence(agentId: string): number {
1916
+ return this.transaction(() => {
1917
+ const row = this.db.prepare(
1918
+ 'SELECT MAX(sequence) AS max_seq FROM engrams WHERE agent_id = ?'
1919
+ ).get(agentId) as { max_seq: number | null };
1920
+ return (row?.max_seq ?? 0) + 1;
1921
+ });
1922
+ }
1923
+
1924
+ /**
1925
+ * Extract the value following a tag prefix, e.g. `["weight=8"]` + `"weight="`
1926
+ * → `"8"`. Returns null if no tag with that prefix is present.
1927
+ */
1928
+ private extractTagValue(tags: string[], tagKeyPrefix: string): string | null {
1929
+ for (const t of tags) {
1930
+ if (t.startsWith(tagKeyPrefix)) {
1931
+ return t.slice(tagKeyPrefix.length);
1932
+ }
1933
+ }
1934
+ return null;
1935
+ }
1936
+ }