agent-working-memory 0.17.0 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +321 -319
  2. package/dist/adapters/common.d.ts +34 -0
  3. package/dist/adapters/common.d.ts.map +1 -1
  4. package/dist/adapters/common.js +528 -446
  5. package/dist/adapters/common.js.map +1 -1
  6. package/dist/api/routes.d.ts.map +1 -1
  7. package/dist/api/routes.js +19 -0
  8. package/dist/api/routes.js.map +1 -1
  9. package/dist/cli/migrate.d.ts.map +1 -1
  10. package/dist/cli/migrate.js +31 -29
  11. package/dist/cli/migrate.js.map +1 -1
  12. package/dist/core/asked-value.d.ts +37 -0
  13. package/dist/core/asked-value.d.ts.map +1 -0
  14. package/dist/core/asked-value.js +77 -0
  15. package/dist/core/asked-value.js.map +1 -0
  16. package/dist/core/conflicts.d.ts +89 -0
  17. package/dist/core/conflicts.d.ts.map +1 -0
  18. package/dist/core/conflicts.js +267 -0
  19. package/dist/core/conflicts.js.map +1 -0
  20. package/dist/core/embeddings.d.ts +14 -2
  21. package/dist/core/embeddings.d.ts.map +1 -1
  22. package/dist/core/embeddings.js +32 -10
  23. package/dist/core/embeddings.js.map +1 -1
  24. package/dist/core/format-recall.d.ts +21 -1
  25. package/dist/core/format-recall.d.ts.map +1 -1
  26. package/dist/core/format-recall.js +60 -5
  27. package/dist/core/format-recall.js.map +1 -1
  28. package/dist/core/ml-worker.d.ts +24 -1
  29. package/dist/core/ml-worker.d.ts.map +1 -1
  30. package/dist/core/ml-worker.js +165 -4
  31. package/dist/core/ml-worker.js.map +1 -1
  32. package/dist/core/model-daemon-client.d.ts +77 -0
  33. package/dist/core/model-daemon-client.d.ts.map +1 -0
  34. package/dist/core/model-daemon-client.js +383 -0
  35. package/dist/core/model-daemon-client.js.map +1 -0
  36. package/dist/core/model-daemon-server.d.ts +71 -0
  37. package/dist/core/model-daemon-server.d.ts.map +1 -0
  38. package/dist/core/model-daemon-server.js +363 -0
  39. package/dist/core/model-daemon-server.js.map +1 -0
  40. package/dist/core/model-daemon.d.ts +137 -0
  41. package/dist/core/model-daemon.d.ts.map +1 -0
  42. package/dist/core/model-daemon.js +231 -0
  43. package/dist/core/model-daemon.js.map +1 -0
  44. package/dist/core/query-expander.d.ts +5 -0
  45. package/dist/core/query-expander.d.ts.map +1 -1
  46. package/dist/core/query-expander.js +3 -2
  47. package/dist/core/query-expander.js.map +1 -1
  48. package/dist/core/recall-config.d.ts +1 -1
  49. package/dist/core/recall-config.d.ts.map +1 -1
  50. package/dist/core/recall-config.js +5 -0
  51. package/dist/core/recall-config.js.map +1 -1
  52. package/dist/core/rerank-window.d.ts +7 -2
  53. package/dist/core/rerank-window.d.ts.map +1 -1
  54. package/dist/core/rerank-window.js +9 -4
  55. package/dist/core/rerank-window.js.map +1 -1
  56. package/dist/core/rerank2.d.ts +52 -2
  57. package/dist/core/rerank2.d.ts.map +1 -1
  58. package/dist/core/rerank2.js +60 -3
  59. package/dist/core/rerank2.js.map +1 -1
  60. package/dist/core/reranker.d.ts +19 -0
  61. package/dist/core/reranker.d.ts.map +1 -1
  62. package/dist/core/reranker.js +76 -12
  63. package/dist/core/reranker.js.map +1 -1
  64. package/dist/core/trust.d.ts +32 -0
  65. package/dist/core/trust.d.ts.map +1 -0
  66. package/dist/core/trust.js +66 -0
  67. package/dist/core/trust.js.map +1 -0
  68. package/dist/core/validity.d.ts +21 -0
  69. package/dist/core/validity.d.ts.map +1 -0
  70. package/dist/core/validity.js +41 -0
  71. package/dist/core/validity.js.map +1 -0
  72. package/dist/core/whoami.d.ts +3 -0
  73. package/dist/core/whoami.d.ts.map +1 -1
  74. package/dist/core/whoami.js +14 -0
  75. package/dist/core/whoami.js.map +1 -1
  76. package/dist/core/write-pipeline.d.ts +48 -0
  77. package/dist/core/write-pipeline.d.ts.map +1 -1
  78. package/dist/core/write-pipeline.js +80 -3
  79. package/dist/core/write-pipeline.js.map +1 -1
  80. package/dist/engine/activation.d.ts.map +1 -1
  81. package/dist/engine/activation.js +27 -4
  82. package/dist/engine/activation.js.map +1 -1
  83. package/dist/engine/consolidation.d.ts.map +1 -1
  84. package/dist/engine/consolidation.js +53 -4
  85. package/dist/engine/consolidation.js.map +1 -1
  86. package/dist/hooks/prime-query.d.ts +57 -0
  87. package/dist/hooks/prime-query.d.ts.map +1 -0
  88. package/dist/hooks/prime-query.js +388 -0
  89. package/dist/hooks/prime-query.js.map +1 -0
  90. package/dist/hooks/prime.d.ts +8 -0
  91. package/dist/hooks/prime.d.ts.map +1 -1
  92. package/dist/hooks/prime.js +21 -2
  93. package/dist/hooks/prime.js.map +1 -1
  94. package/dist/hooks/sidecar.d.ts +7 -0
  95. package/dist/hooks/sidecar.d.ts.map +1 -1
  96. package/dist/hooks/sidecar.js +10 -2
  97. package/dist/hooks/sidecar.js.map +1 -1
  98. package/dist/index.js +11 -3
  99. package/dist/index.js.map +1 -1
  100. package/dist/mcp.js +148 -103
  101. package/dist/mcp.js.map +1 -1
  102. package/dist/model-daemon.d.ts +8 -0
  103. package/dist/model-daemon.d.ts.map +1 -0
  104. package/dist/model-daemon.js +45 -0
  105. package/dist/model-daemon.js.map +1 -0
  106. package/dist/storage/factory.d.ts.map +1 -1
  107. package/dist/storage/factory.js +5 -0
  108. package/dist/storage/factory.js.map +1 -1
  109. package/dist/storage/pglite-schema.d.ts.map +1 -1
  110. package/dist/storage/pglite-schema.js +176 -170
  111. package/dist/storage/pglite-schema.js.map +1 -1
  112. package/dist/storage/pglite.d.ts +5 -0
  113. package/dist/storage/pglite.d.ts.map +1 -1
  114. package/dist/storage/pglite.js +165 -153
  115. package/dist/storage/pglite.js.map +1 -1
  116. package/dist/storage/postgres.d.ts +5 -0
  117. package/dist/storage/postgres.d.ts.map +1 -1
  118. package/dist/storage/postgres.js +165 -153
  119. package/dist/storage/postgres.js.map +1 -1
  120. package/dist/storage/sqlite.d.ts +22 -0
  121. package/dist/storage/sqlite.d.ts.map +1 -1
  122. package/dist/storage/sqlite.js +400 -363
  123. package/dist/storage/sqlite.js.map +1 -1
  124. package/dist/types/engram.d.ts +33 -0
  125. package/dist/types/engram.d.ts.map +1 -1
  126. package/dist/types/engram.js.map +1 -1
  127. package/package.json +1 -1
  128. package/plugin/.claude-plugin/plugin.json +1 -1
  129. package/plugin/README.md +1 -1
  130. package/plugin/bin/awm-mcp-launcher.cjs +2 -2
  131. package/plugin/skills/awm-memory/SKILL.md +11 -2
  132. package/src/adapters/common.ts +881 -802
  133. package/src/api/routes.ts +20 -0
  134. package/src/cli/migrate.ts +309 -307
  135. package/src/core/asked-value.ts +74 -0
  136. package/src/core/conflicts.ts +280 -0
  137. package/src/core/embeddings.ts +35 -10
  138. package/src/core/format-recall.ts +83 -25
  139. package/src/core/ml-worker.ts +458 -281
  140. package/src/core/model-daemon-client.ts +374 -0
  141. package/src/core/model-daemon-server.ts +344 -0
  142. package/src/core/model-daemon.ts +243 -0
  143. package/src/core/query-expander.ts +127 -126
  144. package/src/core/recall-config.ts +5 -0
  145. package/src/core/rerank-window.ts +164 -159
  146. package/src/core/rerank2.ts +143 -82
  147. package/src/core/reranker.ts +78 -11
  148. package/src/core/trust.ts +65 -0
  149. package/src/core/validity.ts +47 -0
  150. package/src/core/whoami.ts +136 -120
  151. package/src/core/write-pipeline.ts +119 -3
  152. package/src/engine/activation.ts +27 -4
  153. package/src/engine/consolidation.ts +950 -899
  154. package/src/hooks/prime-query.ts +375 -0
  155. package/src/hooks/prime.ts +25 -2
  156. package/src/hooks/sidecar.ts +13 -3
  157. package/src/index.ts +255 -248
  158. package/src/mcp.ts +1521 -1475
  159. package/src/model-daemon.ts +48 -0
  160. package/src/storage/factory.ts +25 -19
  161. package/src/storage/pglite-schema.ts +199 -193
  162. package/src/storage/pglite.ts +1513 -1498
  163. package/src/storage/postgres.ts +1613 -1598
  164. package/src/storage/sqlite.ts +2056 -2019
  165. package/src/types/engram.ts +30 -0
@@ -1,2019 +1,2056 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * SQLite storage layer — persistence for engrams, associations, and eval events.
5
- *
6
- * Uses better-sqlite3 for synchronous, fast, embedded storage.
7
- * FTS5 provides BM25 full-text search for the activation pipeline.
8
- */
9
-
10
- import Database from 'better-sqlite3';
11
- import { randomUUID } from 'node:crypto';
12
- import { cosineSimilarity } from '../core/embeddings.js';
13
- import type {
14
- Engram, EngramCreate, EngramStage, Association, AssociationType,
15
- SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
16
- RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
17
- ConsciousState, AutoCheckpoint, CheckpointRow,
18
- } from '../types/index.js';
19
-
20
- /** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
21
- function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
22
- if (buf instanceof ArrayBuffer) return new Float32Array(buf);
23
- // Node Buffer may share an underlying ArrayBuffer — slice to the exact region
24
- const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
25
- return new Float32Array(ab);
26
- }
27
-
28
- const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
29
- surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
30
- };
31
-
32
- /**
33
- * In-memory slim entry — the minimum data the activation pipeline's pre-filter
34
- * pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
35
- * Float32Array conversion on every recall. ~22 bytes overhead per entry plus
36
- * the embedding (~1.5KB), so ~15MB at 10K engrams.
37
- */
38
- type SlimCacheEntry = {
39
- id: string;
40
- agentId: string;
41
- concept: string;
42
- embedding: number[] | null;
43
- stage: EngramStage;
44
- retracted: boolean;
45
- };
46
-
47
- export class EngramStore {
48
- private db: Database.Database;
49
- private walTimer: ReturnType<typeof setInterval> | null = null;
50
-
51
- // Slim cache for the activation pipeline pre-filter. Populated lazily on
52
- // first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
53
- // sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
54
- // (for A/B testing or if a regression appears).
55
- private slimCache: Map<string, SlimCacheEntry> = new Map();
56
- private slimCachePopulated: boolean = false;
57
- private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
58
-
59
- constructor(dbPath: string = 'memory.db') {
60
- this.db = new Database(dbPath);
61
- this.db.pragma('journal_mode = WAL');
62
- this.db.pragma('foreign_keys = ON');
63
- this.db.pragma('busy_timeout = 5000');
64
- this.db.pragma('synchronous = NORMAL');
65
- this.db.pragma('wal_autocheckpoint = 1000');
66
- this.init();
67
- this.startWalCheckpointTimer();
68
- }
69
-
70
- // --- Slim cache management ---
71
-
72
- /** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
73
- private ensureSlimCachePopulated(): void {
74
- if (this.slimCachePopulated || !this.slimCacheEnabled) return;
75
- const rows = this.db.prepare(
76
- 'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
77
- ).all() as any[];
78
- for (const r of rows) {
79
- this.slimCache.set(r.id as string, {
80
- id: r.id as string,
81
- agentId: r.agent_id as string,
82
- concept: r.concept as string,
83
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
84
- stage: r.stage as EngramStage,
85
- retracted: !!r.retracted,
86
- });
87
- }
88
- this.slimCachePopulated = true;
89
- }
90
-
91
- /** Add a new engram to the slim cache. Called from createEngram. */
92
- private cacheAdd(entry: SlimCacheEntry): void {
93
- if (!this.slimCacheEnabled) return;
94
- this.slimCache.set(entry.id, entry);
95
- }
96
-
97
- private cacheUpdateStage(id: string, stage: EngramStage): void {
98
- if (!this.slimCacheEnabled) return;
99
- const e = this.slimCache.get(id);
100
- if (e) e.stage = stage;
101
- }
102
-
103
- private cacheUpdateEmbedding(id: string, embedding: number[]): void {
104
- if (!this.slimCacheEnabled) return;
105
- const e = this.slimCache.get(id);
106
- if (e) e.embedding = embedding;
107
- }
108
-
109
- private cacheRetract(id: string): void {
110
- if (!this.slimCacheEnabled) return;
111
- const e = this.slimCache.get(id);
112
- if (e) e.retracted = true;
113
- }
114
-
115
- private cacheRemove(id: string): void {
116
- if (!this.slimCacheEnabled) return;
117
- this.slimCache.delete(id);
118
- }
119
-
120
- /** Reset cache (used by tests + after timeWarp/bulk operations). */
121
- resetSlimCache(): void {
122
- this.slimCache.clear();
123
- this.slimCachePopulated = false;
124
- }
125
-
126
- /**
127
- * Eager slim-cache populate — public entry point so process startup can warm
128
- * the cache before the first user recall. Otherwise the first recall pays
129
- * a ~600ms one-time SQL fetch + embedding deserialization.
130
- */
131
- warmSlimCache(): void {
132
- this.ensureSlimCachePopulated();
133
- }
134
-
135
- /** Inspect cache state — used for diagnostics + tests. */
136
- getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
137
- return {
138
- populated: this.slimCachePopulated,
139
- size: this.slimCache.size,
140
- enabled: this.slimCacheEnabled,
141
- };
142
- }
143
-
144
- /** Expose the raw database handle for the coordination module. */
145
- getDb(): Database.Database {
146
- return this.db;
147
- }
148
-
149
- /** Run PRAGMA quick_check and return true if DB is healthy. */
150
- integrityCheck(): { ok: boolean; result: string } {
151
- try {
152
- const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
153
- const result = rows[0]?.quick_check ?? 'unknown';
154
- return { ok: result === 'ok', result };
155
- } catch (err) {
156
- return { ok: false, result: (err as Error).message };
157
- }
158
- }
159
-
160
- /** Hot backup using SQLite backup API. Returns the backup path. */
161
- backup(destPath: string): void {
162
- this.db.backup(destPath);
163
- }
164
-
165
- /** Flush WAL to main database file. */
166
- walCheckpoint(): void {
167
- try {
168
- this.db.pragma('wal_checkpoint(TRUNCATE)');
169
- } catch {
170
- // Checkpoint can fail if another connection holds the DB; non-fatal
171
- }
172
- }
173
-
174
- /** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
175
- private startWalCheckpointTimer(): void {
176
- this.walTimer = setInterval(() => {
177
- this.walCheckpoint();
178
- }, 5 * 60 * 1000);
179
- this.walTimer.unref();
180
- }
181
-
182
- /** Stop the WAL checkpoint timer (call before close). */
183
- stopWalCheckpointTimer(): void {
184
- if (this.walTimer) {
185
- clearInterval(this.walTimer);
186
- this.walTimer = null;
187
- }
188
- }
189
-
190
- private init(): void {
191
- this.db.exec(`
192
- CREATE TABLE IF NOT EXISTS engrams (
193
- id TEXT PRIMARY KEY,
194
- agent_id TEXT NOT NULL,
195
- concept TEXT NOT NULL,
196
- content TEXT NOT NULL,
197
- embedding BLOB,
198
- confidence REAL NOT NULL DEFAULT 0.5,
199
- salience REAL NOT NULL DEFAULT 0.5,
200
- access_count INTEGER NOT NULL DEFAULT 0,
201
- last_accessed TEXT NOT NULL,
202
- created_at TEXT NOT NULL,
203
- salience_features TEXT NOT NULL DEFAULT '{}',
204
- reason_codes TEXT NOT NULL DEFAULT '[]',
205
- stage TEXT NOT NULL DEFAULT 'active',
206
- ttl INTEGER,
207
- retracted INTEGER NOT NULL DEFAULT 0,
208
- retracted_by TEXT,
209
- retracted_at TEXT,
210
- tags TEXT NOT NULL DEFAULT '[]',
211
- memory_type TEXT NOT NULL DEFAULT 'unclassified'
212
- );
213
-
214
- CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
215
- CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
216
- CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
217
- CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
218
- -- Serves getTopAccessedTags: the recall path's pronoun branch needs the ten
219
- -- most-accessed engrams, and without this it filters then sorts the whole
220
- -- agent partition on every query containing "it", "this" or "that".
221
- CREATE INDEX IF NOT EXISTS idx_engrams_access ON engrams(agent_id, stage, access_count DESC);
222
-
223
- CREATE TABLE IF NOT EXISTS associations (
224
- id TEXT PRIMARY KEY,
225
- from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
226
- to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
227
- weight REAL NOT NULL DEFAULT 0.1,
228
- confidence REAL NOT NULL DEFAULT 0.5,
229
- type TEXT NOT NULL DEFAULT 'hebbian',
230
- activation_count INTEGER NOT NULL DEFAULT 0,
231
- created_at TEXT NOT NULL,
232
- last_activated TEXT NOT NULL
233
- );
234
-
235
- CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
236
- CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
237
- CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
238
-
239
- CREATE TABLE IF NOT EXISTS agents (
240
- id TEXT PRIMARY KEY,
241
- name TEXT NOT NULL,
242
- created_at TEXT NOT NULL,
243
- config TEXT NOT NULL DEFAULT '{}'
244
- );
245
-
246
- -- FTS5 for full-text search (BM25 ranking built in)
247
- CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
248
- concept, content, tags,
249
- content=engrams,
250
- content_rowid=rowid
251
- );
252
-
253
- -- Triggers to keep FTS in sync
254
- CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
255
- INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
256
- END;
257
- CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
258
- INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
259
- END;
260
- CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
261
- INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
262
- INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
263
- END;
264
-
265
- -- Eval event logs
266
- CREATE TABLE IF NOT EXISTS activation_events (
267
- id TEXT PRIMARY KEY,
268
- agent_id TEXT NOT NULL,
269
- timestamp TEXT NOT NULL,
270
- context TEXT NOT NULL,
271
- results_returned INTEGER NOT NULL,
272
- top_score REAL,
273
- latency_ms REAL NOT NULL,
274
- engram_ids TEXT NOT NULL DEFAULT '[]'
275
- );
276
-
277
- CREATE TABLE IF NOT EXISTS staging_events (
278
- engram_id TEXT NOT NULL,
279
- agent_id TEXT NOT NULL,
280
- action TEXT NOT NULL,
281
- resonance_score REAL,
282
- timestamp TEXT NOT NULL,
283
- age_ms INTEGER NOT NULL
284
- );
285
-
286
- CREATE TABLE IF NOT EXISTS retrieval_feedback (
287
- id TEXT PRIMARY KEY,
288
- activation_event_id TEXT,
289
- engram_id TEXT NOT NULL,
290
- useful INTEGER NOT NULL,
291
- context TEXT,
292
- timestamp TEXT NOT NULL
293
- );
294
-
295
- CREATE TABLE IF NOT EXISTS episodes (
296
- id TEXT PRIMARY KEY,
297
- agent_id TEXT NOT NULL,
298
- label TEXT NOT NULL,
299
- embedding BLOB,
300
- engram_count INTEGER NOT NULL DEFAULT 0,
301
- start_time TEXT NOT NULL,
302
- end_time TEXT NOT NULL,
303
- created_at TEXT NOT NULL
304
- );
305
-
306
- CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
307
- CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
308
- `);
309
-
310
- // Migration: add episode_id column if missing
311
- try {
312
- this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
313
- } catch {
314
- this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
315
- }
316
-
317
- // Migration: add task management columns if missing
318
- try {
319
- this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
320
- } catch {
321
- this.db.exec(`
322
- ALTER TABLE engrams ADD COLUMN task_status TEXT;
323
- ALTER TABLE engrams ADD COLUMN task_priority TEXT;
324
- ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
325
- `);
326
- this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
327
- }
328
-
329
- // Migration: add memory_class and supersession columns if missing
330
- try {
331
- this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
332
- } catch {
333
- this.db.exec(`
334
- ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
335
- ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
336
- ALTER TABLE engrams ADD COLUMN supersedes TEXT;
337
- `);
338
- }
339
-
340
- // Migration: add embedding_model for version tracking (prevents drift on model change)
341
- try {
342
- this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
343
- } catch {
344
- this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
345
- }
346
-
347
- // Entity inverted index + alias table (D9, 2026-07-30) — write-time
348
- // bookkeeping only; retrieval unchanged until D11. Idempotent CREATEs.
349
- this.db.exec(`
350
- CREATE TABLE IF NOT EXISTS entity_mentions (
351
- entity TEXT NOT NULL,
352
- engram_id TEXT NOT NULL,
353
- agent_id TEXT NOT NULL,
354
- PRIMARY KEY (entity, engram_id)
355
- );
356
- CREATE INDEX IF NOT EXISTS idx_entity_mentions_agent ON entity_mentions(agent_id, entity);
357
- CREATE TABLE IF NOT EXISTS entity_aliases (
358
- alias TEXT PRIMARY KEY,
359
- entity TEXT NOT NULL
360
- );
361
- `);
362
-
363
- // Migration: memory-spine provenance + temporal validity (D5/D8, 2026-07-30)
364
- try {
365
- this.db.prepare('SELECT origin_class FROM engrams LIMIT 0').get();
366
- } catch {
367
- this.db.exec(`
368
- ALTER TABLE engrams ADD COLUMN origin_class TEXT;
369
- ALTER TABLE engrams ADD COLUMN writer_session TEXT;
370
- ALTER TABLE engrams ADD COLUMN recipe_id TEXT;
371
- ALTER TABLE engrams ADD COLUMN valid_from TEXT;
372
- ALTER TABLE engrams ADD COLUMN valid_to TEXT;
373
- `);
374
- }
375
-
376
- // Migration: add conscious_state table for checkpointing
377
- this.db.exec(`
378
- CREATE TABLE IF NOT EXISTS conscious_state (
379
- agent_id TEXT PRIMARY KEY,
380
- last_write_id TEXT,
381
- last_recall_context TEXT,
382
- last_recall_ids TEXT NOT NULL DEFAULT '[]',
383
- last_activity_at TEXT NOT NULL DEFAULT (datetime('now')),
384
- write_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
385
- recall_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
386
- execution_state TEXT,
387
- checkpoint_at TEXT,
388
- last_consolidation_at TEXT,
389
- last_mini_consolidation_at TEXT,
390
- consolidation_cycle_count INTEGER NOT NULL DEFAULT 0,
391
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
392
- )
393
- `);
394
-
395
- // Migration: add consolidation_cycle_count if missing (existing DBs)
396
- try {
397
- this.db.exec(`ALTER TABLE conscious_state ADD COLUMN consolidation_cycle_count INTEGER NOT NULL DEFAULT 0`);
398
- } catch { /* column already exists */ }
399
-
400
- // Migration: add memory_type column if missing
401
- try {
402
- this.db.prepare('SELECT memory_type FROM engrams LIMIT 0').get();
403
- } catch {
404
- this.db.exec(`ALTER TABLE engrams ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'unclassified'`);
405
- }
406
-
407
- // Migration (0.8 Cluster A): sequence column for story-time / chronology.
408
- // Nullable — existing engrams stay NULL. Partial index keeps the cost low.
409
- try {
410
- this.db.prepare('SELECT sequence FROM engrams LIMIT 0').get();
411
- } catch {
412
- this.db.exec(`ALTER TABLE engrams ADD COLUMN sequence INTEGER`);
413
- this.db.exec(
414
- `CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence
415
- ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL`
416
- );
417
- }
418
-
419
- // Migration (0.8 Cluster A): references_json column for typed cross-record
420
- // links. Wired through HTTP in Cluster D — schema slot only here.
421
- try {
422
- this.db.prepare('SELECT references_json FROM engrams LIMIT 0').get();
423
- } catch {
424
- this.db.exec(`ALTER TABLE engrams ADD COLUMN references_json TEXT`);
425
- }
426
- }
427
-
428
- // --- Engram CRUD ---
429
-
430
- createEngram(input: EngramCreate): Engram {
431
- const now = new Date().toISOString();
432
- const id = randomUUID();
433
- const embeddingBlob = input.embedding
434
- ? Buffer.from(new Float32Array(input.embedding).buffer)
435
- : null;
436
-
437
- this.db.prepare(`
438
- INSERT INTO engrams (id, agent_id, concept, content, embedding, confidence, salience,
439
- access_count, last_accessed, created_at, salience_features, reason_codes, stage, tags, episode_id,
440
- ttl, memory_class, supersedes, task_status, task_priority, blocked_by, memory_type,
441
- sequence, references_json, origin_class, writer_session, recipe_id, valid_from, valid_to)
442
- VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
443
- `).run(
444
- id, input.agentId, input.concept, input.content, embeddingBlob,
445
- input.confidence ?? 0.5,
446
- input.salience ?? 0.5,
447
- now, now,
448
- JSON.stringify(input.salienceFeatures ?? DEFAULT_SALIENCE_FEATURES),
449
- JSON.stringify(input.reasonCodes ?? []),
450
- JSON.stringify(input.tags ?? []),
451
- input.episodeId ?? null,
452
- input.ttl ?? null,
453
- input.memoryClass ?? 'working',
454
- input.supersedes ?? null,
455
- input.taskStatus ?? null,
456
- input.taskPriority ?? null,
457
- input.blockedBy ?? null,
458
- input.memoryType ?? 'unclassified',
459
- input.sequence ?? null,
460
- input.references && input.references.length > 0
461
- ? JSON.stringify(input.references) : null,
462
- input.originClass ?? null,
463
- input.writerSession ?? null,
464
- input.recipeId ?? null,
465
- input.validFrom ?? null,
466
- input.validTo ?? null,
467
- );
468
-
469
- // Add to slim cache (skip if not yet populated — first slim fetch will load it)
470
- if (this.slimCachePopulated) {
471
- this.cacheAdd({
472
- id,
473
- agentId: input.agentId,
474
- concept: input.concept,
475
- embedding: input.embedding ?? null,
476
- stage: 'active',
477
- retracted: false,
478
- });
479
- }
480
-
481
- return this.getEngram(id)!;
482
- }
483
-
484
- getEngram(id: string): Engram | null {
485
- const row = this.db.prepare('SELECT * FROM engrams WHERE id = ?').get(id) as any;
486
- return row ? this.rowToEngram(row) : null;
487
- }
488
-
489
- getEngramsByAgent(agentId: string, stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
490
- let query = 'SELECT * FROM engrams WHERE agent_id = ?';
491
- const params: any[] = [agentId];
492
-
493
- if (stage) {
494
- query += ' AND stage = ?';
495
- params.push(stage);
496
- }
497
- if (!includeRetracted) {
498
- query += ' AND retracted = 0';
499
- }
500
-
501
- return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
502
- }
503
-
504
- /**
505
- * Slim variant that returns only (id, concept, embedding) — the minimum needed
506
- * for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
507
- * survival check). Avoids materializing the content blob, tag JSON, salience
508
- * features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
509
- *
510
- * Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
511
- * engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
512
- * of that is row materialization of fields we don't read in the filter pass.
513
- */
514
- getEngramsByAgentSlim(
515
- agentId: string,
516
- stage?: EngramStage,
517
- includeRetracted: boolean = false
518
- ): Array<{ id: string; concept: string; embedding: number[] | null }> {
519
- if (this.slimCacheEnabled) {
520
- this.ensureSlimCachePopulated();
521
- const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
522
- for (const entry of this.slimCache.values()) {
523
- if (entry.agentId !== agentId) continue;
524
- if (stage && entry.stage !== stage) continue;
525
- if (!includeRetracted && entry.retracted) continue;
526
- result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
527
- }
528
- return result;
529
- }
530
- // Cache disabled — fall back to direct SQL
531
- let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
532
- const params: any[] = [agentId];
533
-
534
- if (stage) {
535
- query += ' AND stage = ?';
536
- params.push(stage);
537
- }
538
- if (!includeRetracted) {
539
- query += ' AND retracted = 0';
540
- }
541
-
542
- return (this.db.prepare(query).all(...params) as any[]).map(r => ({
543
- id: r.id as string,
544
- concept: r.concept as string,
545
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
546
- }));
547
- }
548
-
549
- /** Slim variant for multi-agent (workspace-scoped) pre-filter. */
550
- getEngramsByAgentsSlim(
551
- agentIds: string[],
552
- stage?: EngramStage,
553
- includeRetracted: boolean = false
554
- ): Array<{ id: string; concept: string; embedding: number[] | null }> {
555
- if (agentIds.length === 0) return [];
556
- if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
557
-
558
- if (this.slimCacheEnabled) {
559
- this.ensureSlimCachePopulated();
560
- const agentSet = new Set(agentIds);
561
- const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
562
- for (const entry of this.slimCache.values()) {
563
- if (!agentSet.has(entry.agentId)) continue;
564
- if (stage && entry.stage !== stage) continue;
565
- if (!includeRetracted && entry.retracted) continue;
566
- result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
567
- }
568
- return result;
569
- }
570
-
571
- const placeholders = agentIds.map(() => '?').join(',');
572
- let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
573
- const params: any[] = [...agentIds];
574
-
575
- if (stage) {
576
- query += ' AND stage = ?';
577
- params.push(stage);
578
- }
579
- if (!includeRetracted) {
580
- query += ' AND retracted = 0';
581
- }
582
-
583
- return (this.db.prepare(query).all(...params) as any[]).map(r => ({
584
- id: r.id as string,
585
- concept: r.concept as string,
586
- embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
587
- }));
588
- }
589
-
590
- /**
591
- * Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
592
- * only the survivors that need deep scoring. Chunks IN-clause queries to stay
593
- * under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
594
- */
595
- getEngramsByIds(ids: string[]): Engram[] {
596
- if (ids.length === 0) return [];
597
- const CHUNK = 800;
598
- const result: Engram[] = [];
599
- for (let i = 0; i < ids.length; i += CHUNK) {
600
- const chunk = ids.slice(i, i + CHUNK);
601
- const placeholders = chunk.map(() => '?').join(',');
602
- const rows = this.db.prepare(
603
- `SELECT * FROM engrams WHERE id IN (${placeholders})`
604
- ).all(...chunk) as any[];
605
- for (const r of rows) result.push(this.rowToEngram(r));
606
- }
607
- return result;
608
- }
609
-
610
- /**
611
- * Get engrams across multiple agents (workspace-scoped recall).
612
- * Used when workspace mode is enabled for hive memory sharing.
613
- */
614
- getEngramsByAgents(agentIds: string[], stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
615
- if (agentIds.length === 0) return [];
616
- if (agentIds.length === 1) return this.getEngramsByAgent(agentIds[0], stage, includeRetracted);
617
-
618
- const placeholders = agentIds.map(() => '?').join(',');
619
- let query = `SELECT * FROM engrams WHERE agent_id IN (${placeholders})`;
620
- const params: any[] = [...agentIds];
621
-
622
- if (stage) {
623
- query += ' AND stage = ?';
624
- params.push(stage);
625
- }
626
- if (!includeRetracted) {
627
- query += ' AND retracted = 0';
628
- }
629
-
630
- return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
631
- }
632
-
633
- /**
634
- * Tags of the most-accessed engrams, bounded IN SQL.
635
- *
636
- * Exists because the recall path's pronoun branch used to call
637
- * `getEngramsByAgents(agentIds, 'active')` — hydrating EVERY active engram
638
- * into JS, embeddings and all — then sort by accessCount in memory, keep the
639
- * top ten, and extract five tag words. Measured 2026-10-08 on an
640
- * 11k-retrievable store: 245ms mean / 280ms median whenever it ran, which was
641
- * 2.1% of real prompts (the words that trigger it include "it", "this",
642
- * "that" and "there") against 0.2% of benchmark probes — so it cost real
643
- * users ~62% of a recall and the benchmark never saw it. See
644
- * docs/recall-latency.md.
645
- *
646
- * ONE QUERY PER AGENT, deliberately. `agent_id IN (...)` makes SQLite abandon
647
- * the ordered index walk and sort the whole partition in a temp B-tree:
648
- * measured on the 11k-retrievable store, the IN form costs 86ms against
649
- * **0.03ms** for the equality form, because only the latter can walk
650
- * `idx_engrams_access` and stop at LIMIT. Agent-scoped recall passes a single
651
- * agent and hits the fast path either way; workspace-scoped recall would have
652
- * paid the 86ms on every pronoun query. The global top N is always contained
653
- * in the union of the per-agent top Ns, so merging in JS is exact, not an
654
- * approximation.
655
- *
656
- * Ordering is `access_count DESC, id`. The `id` tiebreak is deliberate: the
657
- * old JS sort broke ties on SQLite's scan order, which is arbitrary, so this
658
- * is both cheaper and more deterministic.
659
- */
660
- getTopAccessedTags(agentIds: string[], topN: number, stage: EngramStage = 'active'): string[][] {
661
- if (agentIds.length === 0 || topN <= 0) return [];
662
- const stmt = this.db.prepare(`
663
- SELECT tags, access_count AS accessCount, id FROM engrams
664
- WHERE agent_id = ? AND stage = ? AND retracted = 0
665
- ORDER BY access_count DESC, id
666
- LIMIT ?
667
- `);
668
- const rows: { tags: string; accessCount: number; id: string }[] = [];
669
- for (const agentId of agentIds) {
670
- rows.push(...(stmt.all(agentId, stage, topN) as { tags: string; accessCount: number; id: string }[]));
671
- }
672
- if (agentIds.length > 1) {
673
- rows.sort((a, b) => (b.accessCount - a.accessCount) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
674
- }
675
- return rows.slice(0, topN).map(r => {
676
- try { return JSON.parse(r.tags) as string[]; } catch { return []; }
677
- });
678
- }
679
-
680
- /**
681
- * BM25 search across multiple agents (workspace-scoped).
682
- */
683
- searchBM25WithRankMultiAgent(agentIds: string[], query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
684
- if (agentIds.length === 0) return [];
685
- if (agentIds.length === 1) return this.searchBM25WithRank(agentIds[0], query, limit);
686
-
687
- const sanitized = query
688
- .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
689
- .split(/\s+/)
690
- .filter(w => w.length > 1)
691
- .map(w => `"${w}"`)
692
- .join(' OR ');
693
-
694
- if (!sanitized) return [];
695
-
696
- // CTE prefilter — see searchBM25WithRank for rationale (567× speedup verified).
697
- try {
698
- const placeholders = agentIds.map(() => '?').join(',');
699
- const innerLimit = Math.max(limit * 5, 50);
700
- const rows = this.db.prepare(`
701
- WITH top_fts AS (
702
- SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
703
- )
704
- SELECT e.*, top_fts.rank FROM top_fts
705
- JOIN engrams e ON e.rowid = top_fts.rowid
706
- WHERE e.agent_id IN (${placeholders}) AND e.retracted = 0
707
- ORDER BY top_fts.rank
708
- LIMIT ?
709
- `).all(sanitized, innerLimit, ...agentIds, limit) as any[];
710
-
711
- return rows.map(r => ({
712
- engram: this.rowToEngram(r),
713
- bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
714
- }));
715
- } catch {
716
- return [];
717
- }
718
- }
719
-
720
- /**
721
- * Get all distinct agent IDs that share a workspace (requires coord_agents table).
722
- * Returns just the queried agentId if coordination tables don't exist.
723
- */
724
- getWorkspaceAgentIds(agentId: string, workspace: string): string[] {
725
- try {
726
- // Return agent names (not UUIDs) — engrams.agent_id uses name strings
727
- const rows = this.db.prepare(
728
- `SELECT DISTINCT name FROM coord_agents WHERE workspace = ? AND status != 'dead'`
729
- ).all(workspace) as Array<{ name: string }>;
730
- const names = rows.map(r => r.name);
731
- // Ensure the querying agent is always included
732
- if (!names.includes(agentId)) names.push(agentId);
733
- return names;
734
- } catch {
735
- // No coordination tables — fall back to single agent
736
- return [agentId];
737
- }
738
- }
739
-
740
- /**
741
- * Touch an engram: increment access count, update last_accessed, and
742
- * nudge confidence upward. Each retrieval is weak evidence the memory
743
- * is useful — bounded so only explicit feedback can push confidence
744
- * above 0.85. Diminishing returns: first accesses matter most.
745
- *
746
- * Boost: +0.02 per access, scaled by 1/sqrt(accessCount+1), capped at 0.85.
747
- */
748
- touchEngram(id: string): void {
749
- this.db.prepare(`
750
- UPDATE engrams
751
- SET access_count = access_count + 1,
752
- last_accessed = ?,
753
- confidence = MIN(0.85, confidence + 0.02 / (1.0 + sqrt(access_count)))
754
- WHERE id = ?
755
- `).run(new Date().toISOString(), id);
756
- }
757
-
758
- updateStage(id: string, stage: EngramStage): void {
759
- this.db.prepare('UPDATE engrams SET stage = ? WHERE id = ?').run(stage, id);
760
- this.cacheUpdateStage(id, stage);
761
- }
762
-
763
- /**
764
- * Replace an engram's content. Used by the fade phase of consolidation
765
- * (Paper 1: storage degradation) to coarsen un-recalled memories without
766
- * deleting them — concept + tags stay intact for tag-based retrieval, but
767
- * BM25 content surface area shrinks.
768
- */
769
- updateContent(id: string, content: string): void {
770
- this.db.prepare('UPDATE engrams SET content = ? WHERE id = ?').run(content, id);
771
- }
772
-
773
- updateConfidence(id: string, confidence: number): void {
774
- this.db.prepare('UPDATE engrams SET confidence = ? WHERE id = ?').run(
775
- Math.max(0, Math.min(1, confidence)), id
776
- );
777
- }
778
-
779
- updateEmbedding(id: string, embedding: number[], modelId?: string): void {
780
- const blob = Buffer.from(new Float32Array(embedding).buffer);
781
- if (modelId) {
782
- this.db.prepare('UPDATE engrams SET embedding = ?, embedding_model = ? WHERE id = ?').run(blob, modelId, id);
783
- } else {
784
- this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
785
- }
786
- this.cacheUpdateEmbedding(id, embedding);
787
- }
788
-
789
- retractEngram(id: string, retractedBy: string | null): void {
790
- this.db.prepare(`
791
- UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
792
- `).run(retractedBy, new Date().toISOString(), id);
793
- this.cacheRetract(id);
794
- }
795
-
796
- deleteEngram(id: string): void {
797
- this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
798
- this.cacheRemove(id);
799
- }
800
-
801
- /**
802
- * Time warp — shift all timestamps backward by ms milliseconds.
803
- * Used for testing time-dependent behavior (decay, forgetting).
804
- * Returns count of records shifted.
805
- */
806
- timeWarp(agentId: string, ms: number): number {
807
- let count = 0;
808
- const shiftSec = Math.round(ms / 1000);
809
- // Shift engram timestamps
810
- const r1 = this.db.prepare(`
811
- UPDATE engrams SET
812
- created_at = datetime(created_at, '-${shiftSec} seconds'),
813
- last_accessed = datetime(last_accessed, '-${shiftSec} seconds')
814
- WHERE agent_id = ?
815
- `).run(agentId);
816
- count += r1.changes;
817
- // Shift association timestamps
818
- const r2 = this.db.prepare(`
819
- UPDATE associations SET
820
- created_at = datetime(created_at, '-${shiftSec} seconds'),
821
- last_activated = datetime(last_activated, '-${shiftSec} seconds')
822
- WHERE from_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
823
- OR to_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
824
- `).run(agentId, agentId);
825
- count += r2.changes;
826
- return count;
827
- }
828
-
829
- // --- Full-text search (BM25) ---
830
-
831
- searchBM25(agentId: string, query: string, limit: number = 10): Engram[] {
832
- return this.searchBM25WithRank(agentId, query, limit).map(r => r.engram);
833
- }
834
-
835
- /**
836
- * Vector similarity top-K search. SQLite has no native vector index, so we
837
- * iterate over the slim cache, compute cosine similarity in-process, and
838
- * return the top-K nearest neighbors. PGlite uses pgvector + ivfflat for
839
- * the same operation in O(log N).
840
- *
841
- * Returns engrams with their cosine distance (0 = identical, 2 = opposite)
842
- * to match the PGlite contract. Activation engine consumes `1 - distance`
843
- * as the cosine similarity score.
844
- */
845
- searchByVector(
846
- agentId: string,
847
- vec: number[],
848
- limit: number = 10,
849
- ): Array<{ engram: Engram; distance: number }> {
850
- if (!vec || vec.length === 0) return [];
851
- // Include both 'active' and 'fading' — faded engrams retain their embedding
852
- // (Paper 1: storage degradation; only content is trimmed) so they should still
853
- // participate in vector recall. Concept + tags + truncated content remain in BM25.
854
- const activeSlim = this.getEngramsByAgentSlim(agentId, 'active', false);
855
- const fadingSlim = this.getEngramsByAgentSlim(agentId, 'fading', false);
856
- const slim = activeSlim.concat(fadingSlim);
857
- const scored: Array<{ id: string; sim: number }> = [];
858
- for (const e of slim) {
859
- if (!e.embedding) continue;
860
- const sim = cosineSimilarity(vec, e.embedding);
861
- scored.push({ id: e.id, sim });
862
- }
863
- scored.sort((a, b) => b.sim - a.sim);
864
- const topIds = scored.slice(0, limit).map(s => s.id);
865
- if (topIds.length === 0) return [];
866
- const engrams = this.getEngramsByIds(topIds);
867
- const engramMap = new Map(engrams.map(e => [e.id, e]));
868
- return scored
869
- .slice(0, limit)
870
- .map(s => ({ engram: engramMap.get(s.id), sim: s.sim }))
871
- .filter((x): x is { engram: Engram; sim: number } => x.engram !== undefined)
872
- .map(x => ({ engram: x.engram, distance: 1 - x.sim }));
873
- }
874
-
875
- /**
876
- * BM25 search returning rank scores alongside engrams.
877
- * FTS5 rank is negative (lower = better match).
878
- * We normalize to 0-1 where higher = better.
879
- */
880
- searchBM25WithRank(agentId: string, query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
881
- // Sanitize query for FTS5: quote each word to prevent column name interpretation
882
- const sanitized = query
883
- .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
884
- .split(/\s+/)
885
- .filter(w => w.length > 1)
886
- .map(w => `"${w}"`)
887
- .join(' OR ');
888
-
889
- if (!sanitized) return [];
890
-
891
- // CTE prefilter: force FTS5 to apply LIMIT before joining engrams.
892
- //
893
- // Why: the obvious query (JOIN engrams_fts ON rowid + WHERE MATCH + ORDER BY rank LIMIT N)
894
- // makes SQLite's planner materialize ALL matching FTS rows joined with engrams
895
- // before applying LIMIT. With wide OR queries on a 17K-engram index, that's
896
- // thousands of row materializations including 1.5KB embedding blobs — measured
897
- // at 3682ms for a 5-term OR query.
898
- //
899
- // The CTE forces FTS5 to LIMIT first (sub-ms), then join only the top-K rowids.
900
- // Same query plan, 567× faster (3682ms → 6ms verified on 17K engrams).
901
- //
902
- // The inner LIMIT (limit * 5) over-fetches because the agent_id + retracted
903
- // filter is applied AFTER the CTE. limit*5 gives enough headroom that filtered
904
- // results still satisfy the outer LIMIT for typical workloads (single agent
905
- // dominant, low retracted rate).
906
- try {
907
- const innerLimit = Math.max(limit * 5, 50);
908
- const rows = this.db.prepare(`
909
- WITH top_fts AS (
910
- SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
911
- )
912
- SELECT e.*, top_fts.rank FROM top_fts
913
- JOIN engrams e ON e.rowid = top_fts.rowid
914
- WHERE e.agent_id = ? AND e.retracted = 0
915
- ORDER BY top_fts.rank
916
- LIMIT ?
917
- `).all(sanitized, innerLimit, agentId, limit) as any[];
918
-
919
- return rows.map(r => ({
920
- engram: this.rowToEngram(r),
921
- // Normalize: rank is negative, more negative = better match.
922
- // |rank| / (1 + |rank|) gives 0-1 where higher = better.
923
- bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
924
- }));
925
- } catch {
926
- return [];
927
- }
928
- }
929
-
930
- // --- Diagnostic search (deterministic, not cognitive) ---
931
-
932
- search(query: SearchQuery): Engram[] {
933
- let sql = 'SELECT * FROM engrams WHERE agent_id = ?';
934
- const params: any[] = [query.agentId];
935
-
936
- if (query.text) {
937
- sql += ' AND (content LIKE ? OR concept LIKE ?)';
938
- params.push(`%${query.text}%`, `%${query.text}%`);
939
- }
940
- if (query.concept) {
941
- sql += ' AND concept = ?';
942
- params.push(query.concept);
943
- }
944
- if (query.stage) {
945
- sql += ' AND stage = ?';
946
- params.push(query.stage);
947
- }
948
- if (query.retracted !== undefined) {
949
- sql += ' AND retracted = ?';
950
- params.push(query.retracted ? 1 : 0);
951
- }
952
-
953
- // ── Tag filters (0.8 Cluster B) ──
954
- // AND-set: merge legacy `tags` + new `tagsAll`. Both apply (intersection).
955
- const allTags = [...(query.tags ?? []), ...(query.tagsAll ?? [])];
956
- if (allTags.length > 0) {
957
- for (const tag of allTags) {
958
- sql += ' AND tags LIKE ?';
959
- params.push(`%"${tag}"%`);
960
- }
961
- }
962
- // OR-set: at least one must match.
963
- if (query.tagsAny && query.tagsAny.length > 0) {
964
- const ors = query.tagsAny.map(() => 'tags LIKE ?').join(' OR ');
965
- sql += ` AND (${ors})`;
966
- for (const tag of query.tagsAny) params.push(`%"${tag}"%`);
967
- }
968
- // NOT-set: none may match.
969
- if (query.tagsNone && query.tagsNone.length > 0) {
970
- const ors = query.tagsNone.map(() => 'tags LIKE ?').join(' OR ');
971
- sql += ` AND NOT (${ors})`;
972
- for (const tag of query.tagsNone) params.push(`%"${tag}"%`);
973
- }
974
-
975
- // ── Sort (0.8 Cluster B) ──
976
- // Default preserves legacy behavior (last_accessed DESC). Map TS camelCase
977
- // to SQL snake_case. `sequence` puts NULLs last via (col IS NULL) trick.
978
- const sortCol = ({
979
- createdAt: 'created_at',
980
- sequence: 'sequence',
981
- salience: 'salience',
982
- confidence: 'confidence',
983
- lastAccessed: 'last_accessed',
984
- } as const)[query.sortBy ?? 'lastAccessed'];
985
- const dir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
986
- if (query.sortBy === 'sequence') {
987
- // NULLs last regardless of direction so engrams without a story-time
988
- // value don't shuffle into the middle of sequenced results.
989
- sql += ` ORDER BY (sequence IS NULL), sequence ${dir}`;
990
- } else {
991
- sql += ` ORDER BY ${sortCol} ${dir}`;
992
- }
993
-
994
- sql += ` LIMIT ? OFFSET ?`;
995
- params.push(query.limit ?? 50, query.offset ?? 0);
996
-
997
- return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
998
- }
999
-
1000
- /**
1001
- * Get the most recently created engram for an agent (for temporal adjacency edges).
1002
- */
1003
- getLatestEngram(agentId: string, excludeId?: string): Engram | null {
1004
- let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0';
1005
- const params: any[] = [agentId];
1006
- if (excludeId) {
1007
- sql += ' AND id != ?';
1008
- params.push(excludeId);
1009
- }
1010
- // FIX 2026-08-23: created_at is millisecond-resolution ISO text, so two engrams
1011
- // written in the same millisecond TIE and "most recent" became whichever row the
1012
- // engine happened to return. Surfaced on Linux (same-ms writes) while passing on
1013
- // Windows, whose coarser timer separated them. rowid is monotonic with insert
1014
- // order, so it is both deterministic AND chronologically correct as a tiebreak.
1015
- sql += ' ORDER BY created_at DESC, rowid DESC LIMIT 1';
1016
- const row = this.db.prepare(sql).get(...params) as any;
1017
- return row ? this.rowToEngram(row) : null;
1018
- }
1019
-
1020
- // --- Task management ---
1021
-
1022
- updateTaskStatus(id: string, status: TaskStatus): void {
1023
- this.db.prepare('UPDATE engrams SET task_status = ? WHERE id = ?').run(status, id);
1024
- }
1025
-
1026
- updateTaskPriority(id: string, priority: TaskPriority): void {
1027
- this.db.prepare('UPDATE engrams SET task_priority = ? WHERE id = ?').run(priority, id);
1028
- }
1029
-
1030
- updateBlockedBy(id: string, blockedBy: string | null): void {
1031
- this.db.prepare('UPDATE engrams SET blocked_by = ?, task_status = ? WHERE id = ?')
1032
- .run(blockedBy, blockedBy ? 'blocked' : 'open', id);
1033
- }
1034
-
1035
- /**
1036
- * Get tasks for an agent, optionally filtered by status.
1037
- * Results ordered by priority (urgent > high > medium > low), then creation date.
1038
- */
1039
- getTasks(agentId: string, status?: TaskStatus): Engram[] {
1040
- let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND task_status IS NOT NULL AND retracted = 0';
1041
- const params: any[] = [agentId];
1042
- if (status) {
1043
- sql += ' AND task_status = ?';
1044
- params.push(status);
1045
- }
1046
- sql += ` ORDER BY
1047
- CASE task_priority
1048
- WHEN 'urgent' THEN 0
1049
- WHEN 'high' THEN 1
1050
- WHEN 'medium' THEN 2
1051
- WHEN 'low' THEN 3
1052
- ELSE 4
1053
- END,
1054
- created_at DESC`;
1055
- return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
1056
- }
1057
-
1058
- /**
1059
- * Get the next actionable task — highest priority that's not blocked or done.
1060
- */
1061
- getNextTask(agentId: string): Engram | null {
1062
- const row = this.db.prepare(`
1063
- SELECT * FROM engrams
1064
- WHERE agent_id = ? AND task_status IN ('open', 'in_progress') AND retracted = 0
1065
- ORDER BY
1066
- CASE task_status WHEN 'in_progress' THEN 0 ELSE 1 END,
1067
- CASE task_priority
1068
- WHEN 'urgent' THEN 0
1069
- WHEN 'high' THEN 1
1070
- WHEN 'medium' THEN 2
1071
- WHEN 'low' THEN 3
1072
- ELSE 4
1073
- END,
1074
- created_at ASC
1075
- LIMIT 1
1076
- `).get(agentId) as any;
1077
- return row ? this.rowToEngram(row) : null;
1078
- }
1079
-
1080
- // --- Supersession ---
1081
-
1082
- /**
1083
- * Mark an engram as superseded by another.
1084
- * The old memory stays in the DB (historical) but gets down-ranked in recall.
1085
- */
1086
- supersedeEngram(oldId: string, newId: string): void {
1087
- this.db.prepare('UPDATE engrams SET superseded_by = ? WHERE id = ?').run(newId, oldId);
1088
- this.db.prepare('UPDATE engrams SET supersedes = ? WHERE id = ?').run(oldId, newId);
1089
- }
1090
-
1091
- /**
1092
- * Find the most recent active engram matching `concept` (case-insensitive,
1093
- * trimmed) and all of `requiredTags`. Excludes superseded and retracted
1094
- * engrams. Used by /memory/supersede Form B (0.8 Cluster D) and by
1095
- * /memory/write's references[] resolution.
1096
- *
1097
- * Returns null if no match.
1098
- */
1099
- findActiveMatchByConcept(
1100
- agentId: string,
1101
- concept: string,
1102
- requiredTags?: string[],
1103
- ): Engram | null {
1104
- let sql = `SELECT * FROM engrams
1105
- WHERE agent_id = ?
1106
- AND LOWER(TRIM(concept)) = LOWER(TRIM(?))
1107
- AND stage = 'active'
1108
- AND retracted = 0
1109
- AND superseded_by IS NULL`;
1110
- const params: any[] = [agentId, concept];
1111
- if (requiredTags && requiredTags.length > 0) {
1112
- for (const tag of requiredTags) {
1113
- sql += ' AND tags LIKE ?';
1114
- params.push(`%"${tag}"%`);
1115
- }
1116
- }
1117
- // FIX 2026-08-23: created_at is millisecond-resolution ISO text, so two engrams
1118
- // written in the same millisecond TIE and "most recent" became whichever row the
1119
- // engine happened to return. Surfaced on Linux (same-ms writes) while passing on
1120
- // Windows, whose coarser timer separated them. rowid is monotonic with insert
1121
- // order, so it is both deterministic AND chronologically correct as a tiebreak.
1122
- sql += ' ORDER BY created_at DESC, rowid DESC LIMIT 1';
1123
- const row = this.db.prepare(sql).get(...params) as any;
1124
- return row ? this.rowToEngram(row) : null;
1125
- }
1126
-
1127
- /**
1128
- * Check if an engram has been superseded.
1129
- */
1130
- isSuperseded(id: string): boolean {
1131
- const row = this.db.prepare('SELECT superseded_by FROM engrams WHERE id = ?').get(id) as any;
1132
- return row?.superseded_by != null;
1133
- }
1134
-
1135
- updateMemoryClass(id: string, memoryClass: MemoryClass): void {
1136
- this.db.prepare('UPDATE engrams SET memory_class = ? WHERE id = ?').run(memoryClass, id);
1137
- }
1138
-
1139
- updateTags(id: string, tags: string[]): void {
1140
- this.db.prepare('UPDATE engrams SET tags = ? WHERE id = ?').run(JSON.stringify(tags), id);
1141
- }
1142
-
1143
- // --- Associations ---
1144
-
1145
- upsertAssociation(
1146
- fromId: string, toId: string, weight: number,
1147
- type: AssociationType = 'hebbian', confidence: number = 0.5
1148
- ): Association {
1149
- const now = new Date().toISOString();
1150
- const id = randomUUID();
1151
-
1152
- this.db.prepare(`
1153
- INSERT INTO associations (id, from_engram_id, to_engram_id, weight, confidence, type, activation_count, created_at, last_activated)
1154
- VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)
1155
- ON CONFLICT(from_engram_id, to_engram_id) DO UPDATE SET
1156
- weight = ?, confidence = ?, last_activated = ?, activation_count = activation_count + 1
1157
- `).run(id, fromId, toId, weight, confidence, type, now, now, weight, confidence, now);
1158
-
1159
- return this.getAssociation(fromId, toId)!;
1160
- }
1161
-
1162
- getAssociation(fromId: string, toId: string): Association | null {
1163
- const row = this.db.prepare(
1164
- 'SELECT * FROM associations WHERE from_engram_id = ? AND to_engram_id = ?'
1165
- ).get(fromId, toId) as any;
1166
- return row ? this.rowToAssociation(row) : null;
1167
- }
1168
-
1169
- getAssociationsFor(engramId: string): Association[] {
1170
- const rows = this.db.prepare(
1171
- 'SELECT * FROM associations WHERE from_engram_id = ? OR to_engram_id = ?'
1172
- ).all(engramId, engramId);
1173
- return (rows as any[]).map(r => this.rowToAssociation(r));
1174
- }
1175
-
1176
- /**
1177
- * Aggregate association stats per engram — count of edges and sum of weights.
1178
- *
1179
- * Why: the activation scoring loop only uses `associations.length` (Hebbian gate)
1180
- * and `sum of weights` (Hebbian mean + centrality). It doesn't read individual
1181
- * association fields. Returning scalar stats avoids materializing thousands of
1182
- * Association objects.
1183
- *
1184
- * Phase-breakdown spike (2026-05-08, post-0.7.10) showed
1185
- * `getAssociationsForBatch` over ~300 survivors took 222ms (25% of recall floor).
1186
- * Stats-only aggregate via a single GROUP BY drops this to ~20ms.
1187
- *
1188
- * Graph walk still needs full Association rows, but it operates on top-N
1189
- * (~30 candidates) — its per-call `getAssociationsFor` is cheap.
1190
- */
1191
- getAssociationStatsForBatch(engramIds: string[]): Map<string, { count: number; sumWeight: number }> {
1192
- const result = new Map<string, { count: number; sumWeight: number }>();
1193
- if (engramIds.length === 0) return result;
1194
-
1195
- const CHUNK = 400;
1196
- for (let i = 0; i < engramIds.length; i += CHUNK) {
1197
- const chunk = engramIds.slice(i, i + CHUNK);
1198
- const placeholders = chunk.map(() => '?').join(',');
1199
- // UNION ALL counts each association once per endpoint that's in the candidate
1200
- // set — same semantics as the existing getAssociationsForBatch which buckets
1201
- // associations under both their from and to engram. Self-loops would be
1202
- // double-counted, but they're rare in practice and the prior code handled them
1203
- // identically.
1204
- const rows = this.db.prepare(
1205
- `SELECT id, SUM(cnt) AS count, SUM(sw) AS sum_weight FROM (
1206
- SELECT from_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE from_engram_id IN (${placeholders})
1207
- UNION ALL
1208
- SELECT to_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE to_engram_id IN (${placeholders})
1209
- )
1210
- WHERE id IN (${placeholders})
1211
- GROUP BY id`
1212
- ).all(...chunk, ...chunk, ...chunk) as Array<{ id: string; count: number; sum_weight: number }>;
1213
-
1214
- for (const r of rows) {
1215
- result.set(r.id, { count: r.count, sumWeight: r.sum_weight });
1216
- }
1217
- }
1218
- // Ensure every requested id has an entry (even zero-edge engrams)
1219
- for (const id of engramIds) {
1220
- if (!result.has(id)) result.set(id, { count: 0, sumWeight: 0 });
1221
- }
1222
- return result;
1223
- }
1224
-
1225
- /**
1226
- * Batch variant of getAssociationsFor — fetches associations for many engrams
1227
- * in a single query, returning a Map keyed by engram id.
1228
- *
1229
- * Why: per-candidate `getAssociationsFor` calls inside the activation scoring
1230
- * loop are an N+1. Measured at 1300ms for 10K candidates (sub-ms per call but
1231
- * accumulating). One IN-clause query reduces this to ~50ms.
1232
- */
1233
- getAssociationsForBatch(engramIds: string[]): Map<string, Association[]> {
1234
- const result = new Map<string, Association[]>();
1235
- if (engramIds.length === 0) return result;
1236
-
1237
- // SQLite's default SQLITE_LIMIT_VARIABLE_NUMBER is 999. Chunk to stay safely below.
1238
- // We bind each id twice (from + to), so chunks of 400 use 800 placeholders.
1239
- const CHUNK = 400;
1240
- for (let i = 0; i < engramIds.length; i += CHUNK) {
1241
- const chunk = engramIds.slice(i, i + CHUNK);
1242
- const placeholders = chunk.map(() => '?').join(',');
1243
- const rows = this.db.prepare(
1244
- `SELECT * FROM associations
1245
- WHERE from_engram_id IN (${placeholders}) OR to_engram_id IN (${placeholders})`
1246
- ).all(...chunk, ...chunk) as any[];
1247
- for (const r of rows) {
1248
- const a = this.rowToAssociation(r);
1249
- // Bucket by both endpoints — getAssociationsFor returns either-direction matches.
1250
- const fromList = result.get(a.fromEngramId) ?? [];
1251
- fromList.push(a);
1252
- result.set(a.fromEngramId, fromList);
1253
- if (a.toEngramId !== a.fromEngramId) {
1254
- const toList = result.get(a.toEngramId) ?? [];
1255
- toList.push(a);
1256
- result.set(a.toEngramId, toList);
1257
- }
1258
- }
1259
- }
1260
- // Ensure every requested id has an entry (even if empty) so callers can
1261
- // .get() without null-checking.
1262
- for (const id of engramIds) {
1263
- if (!result.has(id)) result.set(id, []);
1264
- }
1265
- return result;
1266
- }
1267
-
1268
- getOutgoingAssociations(engramId: string): Association[] {
1269
- const rows = this.db.prepare(
1270
- 'SELECT * FROM associations WHERE from_engram_id = ?'
1271
- ).all(engramId);
1272
- return (rows as any[]).map(r => this.rowToAssociation(r));
1273
- }
1274
-
1275
- countAssociationsFor(engramId: string): number {
1276
- const row = this.db.prepare(
1277
- 'SELECT COUNT(*) as count FROM associations WHERE from_engram_id = ?'
1278
- ).get(engramId) as any;
1279
- return row.count;
1280
- }
1281
-
1282
- getWeakestAssociation(engramId: string): Association | null {
1283
- const row = this.db.prepare(
1284
- 'SELECT * FROM associations WHERE from_engram_id = ? ORDER BY weight ASC LIMIT 1'
1285
- ).get(engramId) as any;
1286
- return row ? this.rowToAssociation(row) : null;
1287
- }
1288
-
1289
- deleteAssociation(id: string): void {
1290
- this.db.prepare('DELETE FROM associations WHERE id = ?').run(id);
1291
- }
1292
-
1293
- getAllAssociations(agentId: string): Association[] {
1294
- const rows = this.db.prepare(`
1295
- SELECT a.* FROM associations a
1296
- JOIN engrams e ON a.from_engram_id = e.id
1297
- WHERE e.agent_id = ?
1298
- `).all(agentId);
1299
- return (rows as any[]).map(r => this.rowToAssociation(r));
1300
- }
1301
-
1302
- // --- Eviction ---
1303
-
1304
- getEvictionCandidates(agentId: string, limit: number): Engram[] {
1305
- // Lowest combined score: low salience + low access + low confidence + oldest
1306
- const rows = this.db.prepare(`
1307
- SELECT * FROM engrams
1308
- WHERE agent_id = ? AND stage = 'active' AND retracted = 0
1309
- ORDER BY (salience * 0.3 + confidence * 0.3 + (CAST(access_count AS REAL) / (access_count + 5)) * 0.2 +
1310
- (1.0 / (1.0 + (julianday('now') - julianday(last_accessed)))) * 0.2) ASC
1311
- LIMIT ?
1312
- `).all(agentId, limit) as any[];
1313
- return rows.map(r => this.rowToEngram(r));
1314
- }
1315
-
1316
- getActiveCount(agentId: string): number {
1317
- const row = this.db.prepare(
1318
- "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'active'"
1319
- ).get(agentId) as any;
1320
- return row.count;
1321
- }
1322
-
1323
- getStagingCount(agentId: string): number {
1324
- const row = this.db.prepare(
1325
- "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'staging'"
1326
- ).get(agentId) as any;
1327
- return row.count;
1328
- }
1329
-
1330
- // --- Staging buffer ---
1331
-
1332
- getExpiredStaging(): Engram[] {
1333
- const now = Date.now();
1334
- const rows = this.db.prepare(`
1335
- SELECT * FROM engrams WHERE stage = 'staging' AND ttl IS NOT NULL
1336
- `).all() as any[];
1337
-
1338
- return rows
1339
- .map(r => this.rowToEngram(r))
1340
- .filter(e => e.ttl && (e.createdAt.getTime() + e.ttl) < now);
1341
- }
1342
-
1343
- // --- Eval event logging ---
1344
-
1345
- logActivationEvent(event: ActivationEvent): void {
1346
- this.db.prepare(`
1347
- INSERT INTO activation_events (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
1348
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1349
- `).run(
1350
- event.id, event.agentId, event.timestamp.toISOString(),
1351
- event.context, event.resultsReturned, event.topScore,
1352
- event.latencyMs, JSON.stringify(event.engramIds)
1353
- );
1354
- }
1355
-
1356
- logStagingEvent(event: StagingEvent): void {
1357
- this.db.prepare(`
1358
- INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
1359
- VALUES (?, ?, ?, ?, ?, ?)
1360
- `).run(
1361
- event.engramId, event.agentId, event.action,
1362
- event.resonanceScore, event.timestamp.toISOString(), event.ageMs
1363
- );
1364
- }
1365
-
1366
- logRetrievalFeedback(activationEventId: string | null, engramId: string, useful: boolean, context: string): void {
1367
- this.db.prepare(`
1368
- INSERT INTO retrieval_feedback (id, activation_event_id, engram_id, useful, context, timestamp)
1369
- VALUES (?, ?, ?, ?, ?, ?)
1370
- `).run(randomUUID(), activationEventId, engramId, useful ? 1 : 0, context, new Date().toISOString());
1371
- }
1372
-
1373
- // --- Eval metrics queries ---
1374
-
1375
- getRetrievalPrecision(agentId: string, windowHours: number = 24): number {
1376
- const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1377
- const row = this.db.prepare(`
1378
- SELECT
1379
- COUNT(CASE WHEN useful = 1 THEN 1 END) as useful_count,
1380
- COUNT(*) as total_count
1381
- FROM retrieval_feedback rf
1382
- LEFT JOIN activation_events ae ON rf.activation_event_id = ae.id
1383
- JOIN engrams e ON rf.engram_id = e.id
1384
- WHERE e.agent_id = ? AND rf.timestamp > ?
1385
- `).get(agentId, since) as any;
1386
-
1387
- return row.total_count > 0 ? row.useful_count / row.total_count : 0;
1388
- }
1389
-
1390
- getStagingMetrics(agentId: string): { promoted: number; discarded: number; expired: number } {
1391
- const row = this.db.prepare(`
1392
- SELECT
1393
- COUNT(CASE WHEN action = 'promoted' THEN 1 END) as promoted,
1394
- COUNT(CASE WHEN action = 'discarded' THEN 1 END) as discarded,
1395
- COUNT(CASE WHEN action = 'expired' THEN 1 END) as expired
1396
- FROM staging_events WHERE agent_id = ?
1397
- `).get(agentId) as any;
1398
- return { promoted: row.promoted, discarded: row.discarded, expired: row.expired };
1399
- }
1400
-
1401
- getActivationStats(agentId: string, windowHours: number = 24): {
1402
- count: number; avgLatencyMs: number; p50LatencyMs: number; p90LatencyMs: number; p95LatencyMs: number;
1403
- } {
1404
- const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1405
- const rows = this.db.prepare(`
1406
- SELECT latency_ms FROM activation_events
1407
- WHERE agent_id = ? AND timestamp > ?
1408
- ORDER BY latency_ms ASC
1409
- `).all(agentId, since) as { latency_ms: number }[];
1410
-
1411
- if (rows.length === 0) return { count: 0, avgLatencyMs: 0, p50LatencyMs: 0, p90LatencyMs: 0, p95LatencyMs: 0 };
1412
-
1413
- // 0.14.3: p50/p90 added. The column mixes warm recalls with cold model loads
1414
- // and stalls (live store: mean 20 s, median 1.6 s, max 38 min), so the mean
1415
- // is not a usable number; percentiles are. Rows are already sorted ASC.
1416
- const pct = (q: number) => rows[Math.min(Math.floor(rows.length * q), rows.length - 1)].latency_ms;
1417
- const total = rows.reduce((s, r) => s + r.latency_ms, 0);
1418
- return {
1419
- count: rows.length,
1420
- avgLatencyMs: total / rows.length,
1421
- p50LatencyMs: pct(0.5),
1422
- p90LatencyMs: pct(0.9),
1423
- p95LatencyMs: pct(0.95),
1424
- };
1425
- }
1426
-
1427
- /**
1428
- * 0.14.3: feedback rows in the window that are JOINED to an activation event.
1429
- * Unlinked rows (activation_event_id IS NULL — all 872 rows written before
1430
- * 0.14.3) are excluded on purpose: this is the recall→use signal, and a row
1431
- * that cannot be traced to a recall does not measure it.
1432
- */
1433
- getLinkedFeedbackStats(agentId: string, windowHours: number = 24 * 7): { total: number; useful: number } {
1434
- const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1435
- const row = this.db.prepare(`
1436
- SELECT COUNT(*) AS total, COUNT(CASE WHEN rf.useful = 1 THEN 1 END) AS useful
1437
- FROM retrieval_feedback rf
1438
- JOIN activation_events ae ON ae.id = rf.activation_event_id
1439
- WHERE ae.agent_id = ? AND rf.timestamp > ?
1440
- `).get(agentId, since) as { total: number; useful: number };
1441
- return { total: row.total, useful: row.useful };
1442
- }
1443
-
1444
- getConsolidatedCount(agentId: string): number {
1445
- const row = this.db.prepare(
1446
- `SELECT COUNT(*) as cnt FROM engrams WHERE agent_id = ? AND stage = 'consolidated'`
1447
- ).get(agentId) as any;
1448
- return row.cnt;
1449
- }
1450
-
1451
- // --- Helpers ---
1452
-
1453
- private rowToEngram(row: any): Engram {
1454
- return {
1455
- id: row.id,
1456
- agentId: row.agent_id,
1457
- concept: row.concept,
1458
- content: row.content,
1459
- embedding: row.embedding
1460
- ? Array.from(bufferToFloat32Array(row.embedding))
1461
- : null,
1462
- confidence: row.confidence,
1463
- salience: row.salience,
1464
- accessCount: row.access_count,
1465
- lastAccessed: new Date(row.last_accessed),
1466
- createdAt: new Date(row.created_at),
1467
- salienceFeatures: JSON.parse(row.salience_features || '{}'),
1468
- reasonCodes: JSON.parse(row.reason_codes || '[]'),
1469
- stage: row.stage as EngramStage,
1470
- ttl: row.ttl,
1471
- retracted: !!row.retracted,
1472
- retractedBy: row.retracted_by,
1473
- retractedAt: row.retracted_at ? new Date(row.retracted_at) : null,
1474
- tags: JSON.parse(row.tags),
1475
- episodeId: row.episode_id ?? null,
1476
- memoryClass: (row.memory_class ?? 'working') as MemoryClass,
1477
- memoryType: (row.memory_type ?? 'unclassified') as MemoryType,
1478
- supersededBy: row.superseded_by ?? null,
1479
- supersedes: row.supersedes ?? null,
1480
- taskStatus: row.task_status ?? null,
1481
- taskPriority: row.task_priority ?? null,
1482
- blockedBy: row.blocked_by ?? null,
1483
- sequence: row.sequence == null ? null : Number(row.sequence),
1484
- references: row.references_json ? JSON.parse(row.references_json) : null,
1485
- originClass: row.origin_class ?? null,
1486
- writerSession: row.writer_session ?? null,
1487
- recipeId: row.recipe_id ?? null,
1488
- validFrom: row.valid_from ?? null,
1489
- validTo: row.valid_to ?? null,
1490
- };
1491
- }
1492
-
1493
- private rowToAssociation(row: any): Association {
1494
- return {
1495
- id: row.id,
1496
- fromEngramId: row.from_engram_id,
1497
- toEngramId: row.to_engram_id,
1498
- weight: row.weight,
1499
- confidence: row.confidence ?? 0.5,
1500
- type: row.type as AssociationType,
1501
- activationCount: row.activation_count ?? 0,
1502
- createdAt: new Date(row.created_at),
1503
- lastActivated: new Date(row.last_activated),
1504
- };
1505
- }
1506
-
1507
- // --- Episodes ---
1508
-
1509
- createEpisode(input: { agentId: string; label: string; embedding?: number[] }): Episode {
1510
- const now = new Date().toISOString();
1511
- const id = randomUUID();
1512
- const embeddingBlob = input.embedding
1513
- ? Buffer.from(new Float32Array(input.embedding).buffer)
1514
- : null;
1515
-
1516
- this.db.prepare(`
1517
- INSERT INTO episodes (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
1518
- VALUES (?, ?, ?, ?, 0, ?, ?, ?)
1519
- `).run(id, input.agentId, input.label, embeddingBlob, now, now, now);
1520
-
1521
- return this.getEpisode(id)!;
1522
- }
1523
-
1524
- getEpisode(id: string): Episode | null {
1525
- const row = this.db.prepare('SELECT * FROM episodes WHERE id = ?').get(id) as any;
1526
- return row ? this.rowToEpisode(row) : null;
1527
- }
1528
-
1529
- getEpisodesByAgent(agentId: string): Episode[] {
1530
- const rows = this.db.prepare(
1531
- 'SELECT * FROM episodes WHERE agent_id = ? ORDER BY end_time DESC'
1532
- ).all(agentId) as any[];
1533
- return rows.map(r => this.rowToEpisode(r));
1534
- }
1535
-
1536
- getActiveEpisode(agentId: string, windowMs: number = 3600_000): Episode | null {
1537
- // Find most recent episode that ended within the time window
1538
- const cutoff = new Date(Date.now() - windowMs).toISOString();
1539
- const row = this.db.prepare(`
1540
- SELECT * FROM episodes WHERE agent_id = ? AND end_time > ?
1541
- ORDER BY end_time DESC LIMIT 1
1542
- `).get(agentId, cutoff) as any;
1543
- return row ? this.rowToEpisode(row) : null;
1544
- }
1545
-
1546
- addEngramToEpisode(engramId: string, episodeId: string): void {
1547
- this.db.prepare('UPDATE engrams SET episode_id = ? WHERE id = ?').run(episodeId, engramId);
1548
- this.db.prepare(`
1549
- UPDATE episodes SET
1550
- engram_count = engram_count + 1,
1551
- end_time = MAX(end_time, ?)
1552
- WHERE id = ?
1553
- `).run(new Date().toISOString(), episodeId);
1554
- }
1555
-
1556
- getEngramsByEpisode(episodeId: string): Engram[] {
1557
- const rows = this.db.prepare(
1558
- 'SELECT * FROM engrams WHERE episode_id = ? AND retracted = 0 ORDER BY created_at ASC'
1559
- ).all(episodeId) as any[];
1560
- return rows.map(r => this.rowToEngram(r));
1561
- }
1562
-
1563
- updateEpisodeEmbedding(id: string, embedding: number[]): void {
1564
- const blob = Buffer.from(new Float32Array(embedding).buffer);
1565
- this.db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(blob, id);
1566
- }
1567
-
1568
- getEpisodeCount(agentId: string): number {
1569
- const row = this.db.prepare(
1570
- 'SELECT COUNT(*) as cnt FROM episodes WHERE agent_id = ?'
1571
- ).get(agentId) as any;
1572
- return row.cnt;
1573
- }
1574
-
1575
- private rowToEpisode(row: any): Episode {
1576
- return {
1577
- id: row.id,
1578
- agentId: row.agent_id,
1579
- label: row.label,
1580
- embedding: row.embedding
1581
- ? Array.from(bufferToFloat32Array(row.embedding))
1582
- : null,
1583
- engramCount: row.engram_count,
1584
- startTime: new Date(row.start_time),
1585
- endTime: new Date(row.end_time),
1586
- createdAt: new Date(row.created_at),
1587
- };
1588
- }
1589
-
1590
- /**
1591
- * Find engrams whose tags contain any of the given tag values.
1592
- * Used for entity-bridge retrieval: given entity tags from top results,
1593
- * find other engrams mentioning the same entities.
1594
- */
1595
- findEngramsByTags(agentId: string, tags: string[], excludeIds?: Set<string>): Engram[] {
1596
- if (tags.length === 0) return [];
1597
-
1598
- // Build OR conditions for tag matching
1599
- const conditions = tags.map(() => 'tags LIKE ?').join(' OR ');
1600
- const params: any[] = [agentId, ...tags.map(t => `%"${t}"%`)];
1601
-
1602
- let sql = `SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0 AND (${conditions})`;
1603
- const rows = this.db.prepare(sql).all(...params) as any[];
1604
-
1605
- const results = rows.map(r => this.rowToEngram(r));
1606
- if (excludeIds) {
1607
- return results.filter(e => !excludeIds.has(e.id));
1608
- }
1609
- return results;
1610
- }
1611
-
1612
- // --- Checkpointing ---
1613
-
1614
- updateAutoCheckpointWrite(agentId: string, engramId: string): void {
1615
- const now = new Date().toISOString();
1616
- this.db.prepare(`
1617
- INSERT INTO conscious_state (agent_id, last_write_id, last_activity_at, write_count_since_consolidation, updated_at)
1618
- VALUES (?, ?, ?, 1, ?)
1619
- ON CONFLICT(agent_id) DO UPDATE SET
1620
- last_write_id = excluded.last_write_id,
1621
- last_activity_at = excluded.last_activity_at,
1622
- write_count_since_consolidation = write_count_since_consolidation + 1,
1623
- updated_at = excluded.updated_at
1624
- `).run(agentId, engramId, now, now);
1625
- }
1626
-
1627
- updateAutoCheckpointRecall(agentId: string, context: string, engramIds: string[]): void {
1628
- const now = new Date().toISOString();
1629
- this.db.prepare(`
1630
- INSERT INTO conscious_state (agent_id, last_recall_context, last_recall_ids, last_activity_at, recall_count_since_consolidation, updated_at)
1631
- VALUES (?, ?, ?, ?, 1, ?)
1632
- ON CONFLICT(agent_id) DO UPDATE SET
1633
- last_recall_context = excluded.last_recall_context,
1634
- last_recall_ids = excluded.last_recall_ids,
1635
- last_activity_at = excluded.last_activity_at,
1636
- recall_count_since_consolidation = recall_count_since_consolidation + 1,
1637
- updated_at = excluded.updated_at
1638
- `).run(agentId, context, JSON.stringify(engramIds), now, now);
1639
- }
1640
-
1641
- touchActivity(agentId: string): void {
1642
- const now = new Date().toISOString();
1643
- this.db.prepare(`
1644
- INSERT INTO conscious_state (agent_id, last_activity_at, updated_at)
1645
- VALUES (?, ?, ?)
1646
- ON CONFLICT(agent_id) DO UPDATE SET
1647
- last_activity_at = excluded.last_activity_at,
1648
- updated_at = excluded.updated_at
1649
- `).run(agentId, now, now);
1650
- }
1651
-
1652
- saveCheckpoint(agentId: string, state: ConsciousState): void {
1653
- const now = new Date().toISOString();
1654
- this.db.prepare(`
1655
- INSERT INTO conscious_state (agent_id, execution_state, checkpoint_at, last_activity_at, updated_at)
1656
- VALUES (?, ?, ?, ?, ?)
1657
- ON CONFLICT(agent_id) DO UPDATE SET
1658
- execution_state = excluded.execution_state,
1659
- checkpoint_at = excluded.checkpoint_at,
1660
- last_activity_at = excluded.last_activity_at,
1661
- updated_at = excluded.updated_at
1662
- `).run(agentId, JSON.stringify(state), now, now, now);
1663
- }
1664
-
1665
- /** Record entity mentions for an engram (D9 inverted index; idempotent). */
1666
- recordEntityMentions(engramId: string, agentId: string, entities: string[]): void {
1667
- if (!entities || entities.length === 0) return;
1668
- const stmt = this.db.prepare('INSERT OR IGNORE INTO entity_mentions (entity, engram_id, agent_id) VALUES (?, ?, ?)');
1669
- for (const entity of entities) stmt.run(entity, engramId, agentId);
1670
- }
1671
-
1672
- /** Indexed entities whose normalized key:value — or a registered alias — contains the term
1673
- * (D11 query-side lookup). Alias matches resolve to their target entity. */
1674
- searchEntities(term: string, limit: number = 8): string[] {
1675
- const t = term.toLowerCase().replace(/[%_]/g, '');
1676
- if (!t) return [];
1677
- const rows = this.db.prepare(`
1678
- SELECT DISTINCT entity FROM (
1679
- SELECT entity FROM entity_mentions WHERE entity LIKE ?
1680
- UNION SELECT entity FROM entity_aliases WHERE alias LIKE ?
1681
- ) LIMIT ?`).all(`%${t}%`, `%${t}%`, limit) as Array<{ entity: string }>;
1682
- return rows.map(r => r.entity);
1683
- }
1684
-
1685
- /** Engram ids mentioning an entity (alias-resolved), optionally agent-scoped. */
1686
- getEngramIdsByEntity(entity: string, agentId?: string): string[] {
1687
- const resolved = (this.db.prepare('SELECT entity FROM entity_aliases WHERE alias = ?')
1688
- .get(entity.toLowerCase()) as { entity: string } | undefined)?.entity ?? entity.toLowerCase();
1689
- const rows = agentId
1690
- ? this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ? AND agent_id = ?').all(resolved, agentId)
1691
- : this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ?').all(resolved);
1692
- return (rows as Array<{ engram_id: string }>).map(r => r.engram_id);
1693
- }
1694
-
1695
- /** Distinct agent ids present in this store (D3 whoami; includes all stages). */
1696
- listAgentIds(): string[] {
1697
- const rows = this.db.prepare('SELECT DISTINCT agent_id FROM engrams ORDER BY agent_id').all() as Array<{ agent_id: string }>;
1698
- return rows.map(r => r.agent_id);
1699
- }
1700
-
1701
- getCheckpoint(agentId: string): CheckpointRow | null {
1702
- const row = this.db.prepare('SELECT * FROM conscious_state WHERE agent_id = ?').get(agentId) as any;
1703
- if (!row) return null;
1704
-
1705
- return {
1706
- agentId: row.agent_id,
1707
- auto: {
1708
- lastWriteId: row.last_write_id ?? null,
1709
- lastRecallContext: row.last_recall_context ?? null,
1710
- lastRecallIds: JSON.parse(row.last_recall_ids || '[]'),
1711
- lastActivityAt: new Date(row.last_activity_at),
1712
- writeCountSinceConsolidation: row.write_count_since_consolidation,
1713
- recallCountSinceConsolidation: row.recall_count_since_consolidation,
1714
- },
1715
- executionState: row.execution_state ? JSON.parse(row.execution_state) : null,
1716
- checkpointAt: row.checkpoint_at ? new Date(row.checkpoint_at) : null,
1717
- lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1718
- lastMiniConsolidationAt: row.last_mini_consolidation_at ? new Date(row.last_mini_consolidation_at) : null,
1719
- updatedAt: new Date(row.updated_at),
1720
- };
1721
- }
1722
-
1723
- markConsolidation(agentId: string, mini: boolean): void {
1724
- const now = new Date().toISOString();
1725
- if (mini) {
1726
- this.db.prepare(`
1727
- UPDATE conscious_state SET last_mini_consolidation_at = ?, updated_at = ? WHERE agent_id = ?
1728
- `).run(now, now, agentId);
1729
- } else {
1730
- this.db.prepare(`
1731
- UPDATE conscious_state SET
1732
- last_consolidation_at = ?,
1733
- last_mini_consolidation_at = ?,
1734
- write_count_since_consolidation = 0,
1735
- recall_count_since_consolidation = 0,
1736
- consolidation_cycle_count = consolidation_cycle_count + 1,
1737
- updated_at = ?
1738
- WHERE agent_id = ?
1739
- `).run(now, now, now, agentId);
1740
- }
1741
- }
1742
-
1743
- getActiveAgents(): Array<{ agentId: string; lastActivityAt: Date; writeCount: number; recallCount: number; lastConsolidationAt: Date | null }> {
1744
- const rows = this.db.prepare('SELECT * FROM conscious_state').all() as any[];
1745
- return rows.map(row => ({
1746
- agentId: row.agent_id,
1747
- lastActivityAt: new Date(row.last_activity_at),
1748
- writeCount: row.write_count_since_consolidation,
1749
- recallCount: row.recall_count_since_consolidation,
1750
- lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1751
- }));
1752
- }
1753
-
1754
- getConsolidationCycleCount(agentId: string): number {
1755
- const row = this.db.prepare(
1756
- 'SELECT consolidation_cycle_count FROM conscious_state WHERE agent_id = ?',
1757
- ).get(agentId) as { consolidation_cycle_count: number } | undefined;
1758
- return row?.consolidation_cycle_count ?? 0;
1759
- }
1760
-
1761
- close(): void {
1762
- this.stopWalCheckpointTimer();
1763
- this.walCheckpoint();
1764
- this.db.close();
1765
- }
1766
-
1767
- /**
1768
- * Run `fn` in a SQL transaction. All writes succeed atomically or roll back
1769
- * on throw. Synchronous (matches better-sqlite3 semantics).
1770
- *
1771
- * Used by 0.8 Cluster D's `/memory/supersede` Form B to wrap find-match +
1772
- * performWrite + supersede + causal-edge + confidence-decay so callers
1773
- * never observe an intermediate state where new engram exists but old
1774
- * isn't yet superseded.
1775
- */
1776
- transaction<T>(fn: () => T): T {
1777
- return this.db.transaction(fn)();
1778
- }
1779
-
1780
- /**
1781
- * Async-aware transaction wrapper. `fn` may await — we hold the SQLite
1782
- * lock across awaits by issuing raw BEGIN/COMMIT/ROLLBACK statements rather
1783
- * than using better-sqlite3's sync `db.transaction()`.
1784
- *
1785
- * Used by Form B (atomic write + supersede) where the body calls
1786
- * async store methods through the IEngramStore contract.
1787
- */
1788
- async withTransaction<T>(fn: () => Promise<T>): Promise<T> {
1789
- this.db.exec('BEGIN');
1790
- try {
1791
- const result = await fn();
1792
- this.db.exec('COMMIT');
1793
- return result;
1794
- } catch (err) {
1795
- try { this.db.exec('ROLLBACK'); } catch { /* best-effort */ }
1796
- throw err;
1797
- }
1798
- }
1799
-
1800
- // ============================================================
1801
- // 0.8 Cluster C — materialized-view + atomic-counter primitives
1802
- // ============================================================
1803
-
1804
- /**
1805
- * For each distinct value of the tag whose key prefix is `tagKeyPrefix`
1806
- * (e.g. `"character="`), return the most-recent active engram. Used by
1807
- * NovelForge's "latest emotional state per character", "latest motif
1808
- * phase per motif", "recent chapter summaries".
1809
- *
1810
- * `sortBy` defaults to `createdAt`. With `"sequence"`, engrams without a
1811
- * sequence are excluded (they have no story-time anchor).
1812
- */
1813
- getLatestByTag(opts: {
1814
- agentId: string;
1815
- tagKeyPrefix: string; // e.g. "character=", "motif=", "chapter="
1816
- scopeTagsAll?: string[]; // optional narrowing (e.g. ["topic=motif-use"])
1817
- retracted?: boolean;
1818
- sortBy?: 'createdAt' | 'sequence';
1819
- limit?: number;
1820
- }): Engram[] {
1821
- let sql = `SELECT * FROM engrams
1822
- WHERE agent_id = ?
1823
- AND retracted = ?
1824
- AND stage = 'active'
1825
- AND tags LIKE ?`;
1826
- const params: any[] = [
1827
- opts.agentId,
1828
- opts.retracted ? 1 : 0,
1829
- `%"${opts.tagKeyPrefix}%`,
1830
- ];
1831
- if (opts.scopeTagsAll && opts.scopeTagsAll.length > 0) {
1832
- for (const t of opts.scopeTagsAll) {
1833
- sql += ' AND tags LIKE ?';
1834
- params.push(`%"${t}"%`);
1835
- }
1836
- }
1837
- if (opts.sortBy === 'sequence') {
1838
- sql += ' AND sequence IS NOT NULL';
1839
- }
1840
- sql += ' ORDER BY ' + (opts.sortBy === 'sequence' ? 'sequence DESC, created_at DESC' : 'created_at DESC');
1841
-
1842
- const rows = this.db.prepare(sql).all(...params) as any[];
1843
- const engrams = rows.map(r => this.rowToEngram(r));
1844
-
1845
- // Group by extracted tag value, take first (already sorted newest first).
1846
- const seen = new Map<string, Engram>();
1847
- for (const e of engrams) {
1848
- const value = this.extractTagValue(e.tags, opts.tagKeyPrefix);
1849
- if (value == null) continue;
1850
- if (!seen.has(value)) seen.set(value, e);
1851
- }
1852
- const out = Array.from(seen.values());
1853
- return opts.limit ? out.slice(0, opts.limit) : out;
1854
- }
1855
-
1856
- /**
1857
- * Filter engrams by tag-set operators and sort by numeric value extracted
1858
- * from a tag prefix (e.g. `"weight="` → numeric value of `weight=8`).
1859
- * Used by NovelForge's "top 40 active promises by weight".
1860
- *
1861
- * Missing or unparseable numeric values sort last.
1862
- */
1863
- getTopBy(opts: {
1864
- agentId: string;
1865
- sortField: string; // tag prefix, e.g. "weight="
1866
- order: 'asc' | 'desc';
1867
- filterTagsAll?: string[];
1868
- filterTagsAny?: string[];
1869
- filterTagsNone?: string[];
1870
- limit?: number;
1871
- retracted?: boolean;
1872
- }): Engram[] {
1873
- // Build the base query inline (mirrors search() but adds the topBy sort).
1874
- let sql = `SELECT * FROM engrams
1875
- WHERE agent_id = ?
1876
- AND retracted = ?
1877
- AND stage = 'active'
1878
- AND tags LIKE ?`;
1879
- const params: any[] = [
1880
- opts.agentId,
1881
- opts.retracted ? 1 : 0,
1882
- `%"${opts.sortField}%`, // must have a tag with this prefix
1883
- ];
1884
-
1885
- if (opts.filterTagsAll && opts.filterTagsAll.length > 0) {
1886
- for (const tag of opts.filterTagsAll) {
1887
- sql += ' AND tags LIKE ?';
1888
- params.push(`%"${tag}"%`);
1889
- }
1890
- }
1891
- if (opts.filterTagsAny && opts.filterTagsAny.length > 0) {
1892
- const ors = opts.filterTagsAny.map(() => 'tags LIKE ?').join(' OR ');
1893
- sql += ` AND (${ors})`;
1894
- for (const tag of opts.filterTagsAny) params.push(`%"${tag}"%`);
1895
- }
1896
- if (opts.filterTagsNone && opts.filterTagsNone.length > 0) {
1897
- const ors = opts.filterTagsNone.map(() => 'tags LIKE ?').join(' OR ');
1898
- sql += ` AND NOT (${ors})`;
1899
- for (const tag of opts.filterTagsNone) params.push(`%"${tag}"%`);
1900
- }
1901
-
1902
- const rows = this.db.prepare(sql).all(...params) as any[];
1903
- const engrams = rows.map(r => this.rowToEngram(r));
1904
-
1905
- // Extract numeric value from the sortField tag; missing or NaN sorts last.
1906
- const valued = engrams.map(e => {
1907
- const raw = this.extractTagValue(e.tags, opts.sortField);
1908
- const n = raw == null ? NaN : Number(raw);
1909
- return { e, n };
1910
- });
1911
- valued.sort((a, b) => {
1912
- const aNaN = Number.isNaN(a.n);
1913
- const bNaN = Number.isNaN(b.n);
1914
- if (aNaN && bNaN) return 0;
1915
- if (aNaN) return 1; // NaN last
1916
- if (bNaN) return -1;
1917
- return opts.order === 'asc' ? a.n - b.n : b.n - a.n;
1918
- });
1919
- const sorted = valued.map(v => v.e);
1920
- return opts.limit ? sorted.slice(0, opts.limit) : sorted;
1921
- }
1922
-
1923
- /**
1924
- * Compute the effective state of an engram from referenced events.
1925
- *
1926
- * - 'superseded' if engram.supersededBy is set
1927
- * - else: scan engrams whose references_json names this engram with a
1928
- * terminal relation type (resolves / subverts / abandons). Take the
1929
- * latest by createdAt. effectiveState = that type.
1930
- * - else: 'active'
1931
- *
1932
- * Returns `null` if target engram not found.
1933
- */
1934
- resolveEffectiveState(targetEngramId: string): {
1935
- engram: Engram;
1936
- effectiveState: 'active' | 'resolved' | 'subverted' | 'abandoned' | 'superseded';
1937
- resolvingEvents: Array<{ id: string; type: string; createdAt: string }>;
1938
- } | null {
1939
- const target = this.getEngram(targetEngramId);
1940
- if (!target) return null;
1941
-
1942
- if (target.supersededBy) {
1943
- return { engram: target, effectiveState: 'superseded', resolvingEvents: [] };
1944
- }
1945
-
1946
- // Find candidate referencing engrams. LIKE-filter first to narrow,
1947
- // then JSON.parse each candidate's references_json to verify the
1948
- // matchEngramId and type. Cheap enough at NovelForge's scale.
1949
- const candidates = this.db.prepare(
1950
- `SELECT id, references_json, created_at FROM engrams
1951
- WHERE agent_id = ?
1952
- AND retracted = 0
1953
- AND stage = 'active'
1954
- AND references_json IS NOT NULL
1955
- AND references_json LIKE ?`
1956
- ).all(target.agentId, `%"${targetEngramId}"%`) as Array<{
1957
- id: string;
1958
- references_json: string;
1959
- created_at: string;
1960
- }>;
1961
-
1962
- const terminalTypes = new Set(['resolves', 'subverts', 'abandons']);
1963
- const resolvingEvents: Array<{ id: string; type: string; createdAt: string }> = [];
1964
- for (const c of candidates) {
1965
- try {
1966
- const refs = JSON.parse(c.references_json) as Array<{ type: string; matchEngramId?: string }>;
1967
- for (const ref of refs) {
1968
- if (ref.matchEngramId === targetEngramId && terminalTypes.has(ref.type)) {
1969
- resolvingEvents.push({ id: c.id, type: ref.type, createdAt: c.created_at });
1970
- }
1971
- }
1972
- } catch { /* malformed references_json — skip */ }
1973
- }
1974
-
1975
- if (resolvingEvents.length === 0) {
1976
- return { engram: target, effectiveState: 'active', resolvingEvents: [] };
1977
- }
1978
-
1979
- // Latest event wins.
1980
- resolvingEvents.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
1981
- const latestType = resolvingEvents[0]!.type as 'resolves' | 'subverts' | 'abandons';
1982
- const effectiveState = (
1983
- latestType === 'resolves' ? 'resolved'
1984
- : latestType === 'subverts' ? 'subverted'
1985
- : 'abandoned'
1986
- );
1987
- return { engram: target, effectiveState, resolvingEvents };
1988
- }
1989
-
1990
- /**
1991
- * Atomically allocate the next sequence number for an agent.
1992
- *
1993
- * Uses BEGIN IMMEDIATE to serialize concurrent callers — better-sqlite3's
1994
- * transaction() acquires a RESERVED lock which prevents other writers
1995
- * from racing. Returns MAX(sequence)+1 for the agent, or 1 if no engrams
1996
- * have a sequence yet.
1997
- */
1998
- allocateNextSequence(agentId: string): number {
1999
- return this.transaction(() => {
2000
- const row = this.db.prepare(
2001
- 'SELECT MAX(sequence) AS max_seq FROM engrams WHERE agent_id = ?'
2002
- ).get(agentId) as { max_seq: number | null };
2003
- return (row?.max_seq ?? 0) + 1;
2004
- });
2005
- }
2006
-
2007
- /**
2008
- * Extract the value following a tag prefix, e.g. `["weight=8"]` + `"weight="`
2009
- * → `"8"`. Returns null if no tag with that prefix is present.
2010
- */
2011
- private extractTagValue(tags: string[], tagKeyPrefix: string): string | null {
2012
- for (const t of tags) {
2013
- if (t.startsWith(tagKeyPrefix)) {
2014
- return t.slice(tagKeyPrefix.length);
2015
- }
2016
- }
2017
- return null;
2018
- }
2019
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * SQLite storage layer — persistence for engrams, associations, and eval events.
5
+ *
6
+ * Uses better-sqlite3 for synchronous, fast, embedded storage.
7
+ * FTS5 provides BM25 full-text search for the activation pipeline.
8
+ */
9
+
10
+ import Database from 'better-sqlite3';
11
+ import { randomUUID } from 'node:crypto';
12
+ import { cosineSimilarity } from '../core/embeddings.js';
13
+ import type {
14
+ Engram, EngramCreate, EngramStage, Association, AssociationType,
15
+ SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
16
+ RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
17
+ ConsciousState, AutoCheckpoint, CheckpointRow,
18
+ } from '../types/index.js';
19
+
20
+ /** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
21
+ function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
22
+ if (buf instanceof ArrayBuffer) return new Float32Array(buf);
23
+ // Node Buffer may share an underlying ArrayBuffer — slice to the exact region
24
+ const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
25
+ return new Float32Array(ab);
26
+ }
27
+
28
+ const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
29
+ surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
30
+ };
31
+
32
+ /**
33
+ * In-memory slim entry — the minimum data the activation pipeline's pre-filter
34
+ * pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
35
+ * Float32Array conversion on every recall. ~22 bytes overhead per entry plus
36
+ * the embedding (~1.5KB), so ~15MB at 10K engrams.
37
+ */
38
+ type SlimCacheEntry = {
39
+ id: string;
40
+ agentId: string;
41
+ concept: string;
42
+ embedding: number[] | null;
43
+ stage: EngramStage;
44
+ retracted: boolean;
45
+ };
46
+
47
+ export class EngramStore {
48
+ private db: Database.Database;
49
+ private walTimer: ReturnType<typeof setInterval> | null = null;
50
+
51
+ // Slim cache for the activation pipeline pre-filter. Populated lazily on
52
+ // first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
53
+ // sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
54
+ // (for A/B testing or if a regression appears).
55
+ private slimCache: Map<string, SlimCacheEntry> = new Map();
56
+ private slimCachePopulated: boolean = false;
57
+ private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
58
+
59
+ constructor(dbPath: string = 'memory.db') {
60
+ this.db = new Database(dbPath);
61
+ this.db.pragma('journal_mode = WAL');
62
+ this.db.pragma('foreign_keys = ON');
63
+ this.db.pragma('busy_timeout = 5000');
64
+ this.db.pragma('synchronous = NORMAL');
65
+ this.db.pragma('wal_autocheckpoint = 1000');
66
+ this.init();
67
+ this.startWalCheckpointTimer();
68
+ }
69
+
70
+ // --- Slim cache management ---
71
+
72
+ /** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
73
+ private ensureSlimCachePopulated(): void {
74
+ if (this.slimCachePopulated || !this.slimCacheEnabled) return;
75
+ const rows = this.db.prepare(
76
+ 'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
77
+ ).all() as any[];
78
+ for (const r of rows) {
79
+ this.slimCache.set(r.id as string, {
80
+ id: r.id as string,
81
+ agentId: r.agent_id as string,
82
+ concept: r.concept as string,
83
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
84
+ stage: r.stage as EngramStage,
85
+ retracted: !!r.retracted,
86
+ });
87
+ }
88
+ this.slimCachePopulated = true;
89
+ }
90
+
91
+ /** Add a new engram to the slim cache. Called from createEngram. */
92
+ private cacheAdd(entry: SlimCacheEntry): void {
93
+ if (!this.slimCacheEnabled) return;
94
+ this.slimCache.set(entry.id, entry);
95
+ }
96
+
97
+ private cacheUpdateStage(id: string, stage: EngramStage): void {
98
+ if (!this.slimCacheEnabled) return;
99
+ const e = this.slimCache.get(id);
100
+ if (e) e.stage = stage;
101
+ }
102
+
103
+ private cacheUpdateEmbedding(id: string, embedding: number[]): void {
104
+ if (!this.slimCacheEnabled) return;
105
+ const e = this.slimCache.get(id);
106
+ if (e) e.embedding = embedding;
107
+ }
108
+
109
+ private cacheRetract(id: string): void {
110
+ if (!this.slimCacheEnabled) return;
111
+ const e = this.slimCache.get(id);
112
+ if (e) e.retracted = true;
113
+ }
114
+
115
+ private cacheRemove(id: string): void {
116
+ if (!this.slimCacheEnabled) return;
117
+ this.slimCache.delete(id);
118
+ }
119
+
120
+ /** Reset cache (used by tests + after timeWarp/bulk operations). */
121
+ resetSlimCache(): void {
122
+ this.slimCache.clear();
123
+ this.slimCachePopulated = false;
124
+ }
125
+
126
+ /**
127
+ * Eager slim-cache populate — public entry point so process startup can warm
128
+ * the cache before the first user recall. Otherwise the first recall pays
129
+ * a ~600ms one-time SQL fetch + embedding deserialization.
130
+ */
131
+ warmSlimCache(): void {
132
+ this.ensureSlimCachePopulated();
133
+ }
134
+
135
+ /** Inspect cache state — used for diagnostics + tests. */
136
+ getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
137
+ return {
138
+ populated: this.slimCachePopulated,
139
+ size: this.slimCache.size,
140
+ enabled: this.slimCacheEnabled,
141
+ };
142
+ }
143
+
144
+ /** Expose the raw database handle for the coordination module. */
145
+ getDb(): Database.Database {
146
+ return this.db;
147
+ }
148
+
149
+ /** Run PRAGMA quick_check and return true if DB is healthy. */
150
+ integrityCheck(): { ok: boolean; result: string } {
151
+ try {
152
+ const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
153
+ const result = rows[0]?.quick_check ?? 'unknown';
154
+ return { ok: result === 'ok', result };
155
+ } catch (err) {
156
+ return { ok: false, result: (err as Error).message };
157
+ }
158
+ }
159
+
160
+ /** Hot backup using SQLite backup API. Returns the backup path. */
161
+ backup(destPath: string): void {
162
+ this.db.backup(destPath);
163
+ }
164
+
165
+ /** Flush WAL to main database file. */
166
+ walCheckpoint(): void {
167
+ try {
168
+ this.db.pragma('wal_checkpoint(TRUNCATE)');
169
+ } catch {
170
+ // Checkpoint can fail if another connection holds the DB; non-fatal
171
+ }
172
+ }
173
+
174
+ /** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
175
+ private startWalCheckpointTimer(): void {
176
+ this.walTimer = setInterval(() => {
177
+ this.walCheckpoint();
178
+ }, 5 * 60 * 1000);
179
+ this.walTimer.unref();
180
+ }
181
+
182
+ /** Stop the WAL checkpoint timer (call before close). */
183
+ stopWalCheckpointTimer(): void {
184
+ if (this.walTimer) {
185
+ clearInterval(this.walTimer);
186
+ this.walTimer = null;
187
+ }
188
+ }
189
+
190
+ private init(): void {
191
+ this.db.exec(`
192
+ CREATE TABLE IF NOT EXISTS engrams (
193
+ id TEXT PRIMARY KEY,
194
+ agent_id TEXT NOT NULL,
195
+ concept TEXT NOT NULL,
196
+ content TEXT NOT NULL,
197
+ embedding BLOB,
198
+ confidence REAL NOT NULL DEFAULT 0.5,
199
+ salience REAL NOT NULL DEFAULT 0.5,
200
+ access_count INTEGER NOT NULL DEFAULT 0,
201
+ last_accessed TEXT NOT NULL,
202
+ created_at TEXT NOT NULL,
203
+ salience_features TEXT NOT NULL DEFAULT '{}',
204
+ reason_codes TEXT NOT NULL DEFAULT '[]',
205
+ stage TEXT NOT NULL DEFAULT 'active',
206
+ ttl INTEGER,
207
+ retracted INTEGER NOT NULL DEFAULT 0,
208
+ retracted_by TEXT,
209
+ retracted_at TEXT,
210
+ tags TEXT NOT NULL DEFAULT '[]',
211
+ memory_type TEXT NOT NULL DEFAULT 'unclassified'
212
+ );
213
+
214
+ CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
215
+ CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
216
+ CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
217
+ CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
218
+ -- Serves getTopAccessedTags: the recall path's pronoun branch needs the ten
219
+ -- most-accessed engrams, and without this it filters then sorts the whole
220
+ -- agent partition on every query containing "it", "this" or "that".
221
+ CREATE INDEX IF NOT EXISTS idx_engrams_access ON engrams(agent_id, stage, access_count DESC);
222
+
223
+ CREATE TABLE IF NOT EXISTS associations (
224
+ id TEXT PRIMARY KEY,
225
+ from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
226
+ to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
227
+ weight REAL NOT NULL DEFAULT 0.1,
228
+ confidence REAL NOT NULL DEFAULT 0.5,
229
+ type TEXT NOT NULL DEFAULT 'hebbian',
230
+ activation_count INTEGER NOT NULL DEFAULT 0,
231
+ created_at TEXT NOT NULL,
232
+ last_activated TEXT NOT NULL
233
+ );
234
+
235
+ CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
236
+ CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
237
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
238
+
239
+ CREATE TABLE IF NOT EXISTS agents (
240
+ id TEXT PRIMARY KEY,
241
+ name TEXT NOT NULL,
242
+ created_at TEXT NOT NULL,
243
+ config TEXT NOT NULL DEFAULT '{}'
244
+ );
245
+
246
+ -- FTS5 for full-text search (BM25 ranking built in)
247
+ CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
248
+ concept, content, tags,
249
+ content=engrams,
250
+ content_rowid=rowid
251
+ );
252
+
253
+ -- Triggers to keep FTS in sync
254
+ CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
255
+ INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
256
+ END;
257
+ CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
258
+ INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
259
+ END;
260
+ CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
261
+ INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
262
+ INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
263
+ END;
264
+
265
+ -- Eval event logs
266
+ CREATE TABLE IF NOT EXISTS activation_events (
267
+ id TEXT PRIMARY KEY,
268
+ agent_id TEXT NOT NULL,
269
+ timestamp TEXT NOT NULL,
270
+ context TEXT NOT NULL,
271
+ results_returned INTEGER NOT NULL,
272
+ top_score REAL,
273
+ latency_ms REAL NOT NULL,
274
+ engram_ids TEXT NOT NULL DEFAULT '[]'
275
+ );
276
+
277
+ CREATE TABLE IF NOT EXISTS staging_events (
278
+ engram_id TEXT NOT NULL,
279
+ agent_id TEXT NOT NULL,
280
+ action TEXT NOT NULL,
281
+ resonance_score REAL,
282
+ timestamp TEXT NOT NULL,
283
+ age_ms INTEGER NOT NULL
284
+ );
285
+
286
+ CREATE TABLE IF NOT EXISTS retrieval_feedback (
287
+ id TEXT PRIMARY KEY,
288
+ activation_event_id TEXT,
289
+ engram_id TEXT NOT NULL,
290
+ useful INTEGER NOT NULL,
291
+ context TEXT,
292
+ timestamp TEXT NOT NULL
293
+ );
294
+
295
+ CREATE TABLE IF NOT EXISTS episodes (
296
+ id TEXT PRIMARY KEY,
297
+ agent_id TEXT NOT NULL,
298
+ label TEXT NOT NULL,
299
+ embedding BLOB,
300
+ engram_count INTEGER NOT NULL DEFAULT 0,
301
+ start_time TEXT NOT NULL,
302
+ end_time TEXT NOT NULL,
303
+ created_at TEXT NOT NULL
304
+ );
305
+
306
+ CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
307
+ CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
308
+ `);
309
+
310
+ // Migration: add episode_id column if missing
311
+ try {
312
+ this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
313
+ } catch {
314
+ this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
315
+ }
316
+
317
+ // Migration: add task management columns if missing
318
+ try {
319
+ this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
320
+ } catch {
321
+ this.db.exec(`
322
+ ALTER TABLE engrams ADD COLUMN task_status TEXT;
323
+ ALTER TABLE engrams ADD COLUMN task_priority TEXT;
324
+ ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
325
+ `);
326
+ this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
327
+ }
328
+
329
+ // Migration: add memory_class and supersession columns if missing
330
+ try {
331
+ this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
332
+ } catch {
333
+ this.db.exec(`
334
+ ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
335
+ ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
336
+ ALTER TABLE engrams ADD COLUMN supersedes TEXT;
337
+ `);
338
+ }
339
+
340
+ // Migration: add embedding_model for version tracking (prevents drift on model change)
341
+ try {
342
+ this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
343
+ } catch {
344
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
345
+ }
346
+
347
+ // Entity inverted index + alias table (D9, 2026-07-30) — write-time
348
+ // bookkeeping only; retrieval unchanged until D11. Idempotent CREATEs.
349
+ this.db.exec(`
350
+ CREATE TABLE IF NOT EXISTS entity_mentions (
351
+ entity TEXT NOT NULL,
352
+ engram_id TEXT NOT NULL,
353
+ agent_id TEXT NOT NULL,
354
+ PRIMARY KEY (entity, engram_id)
355
+ );
356
+ CREATE INDEX IF NOT EXISTS idx_entity_mentions_agent ON entity_mentions(agent_id, entity);
357
+ CREATE TABLE IF NOT EXISTS entity_aliases (
358
+ alias TEXT PRIMARY KEY,
359
+ entity TEXT NOT NULL
360
+ );
361
+ `);
362
+
363
+ // Migration: memory-spine provenance + temporal validity (D5/D8, 2026-07-30)
364
+ try {
365
+ this.db.prepare('SELECT origin_class FROM engrams LIMIT 0').get();
366
+ } catch {
367
+ this.db.exec(`
368
+ ALTER TABLE engrams ADD COLUMN origin_class TEXT;
369
+ ALTER TABLE engrams ADD COLUMN writer_session TEXT;
370
+ ALTER TABLE engrams ADD COLUMN recipe_id TEXT;
371
+ ALTER TABLE engrams ADD COLUMN valid_from TEXT;
372
+ ALTER TABLE engrams ADD COLUMN valid_to TEXT;
373
+ `);
374
+ }
375
+
376
+ // Migration: stage-transition timestamp (A-4, 2026-10-12). Additive, nullable,
377
+ // no backfill — null means "never transitioned", so readers fall back to created_at.
378
+ try {
379
+ this.db.prepare('SELECT stage_changed_at FROM engrams LIMIT 0').get();
380
+ } catch {
381
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN stage_changed_at TEXT`);
382
+ }
383
+
384
+ // Migration: add conscious_state table for checkpointing
385
+ this.db.exec(`
386
+ CREATE TABLE IF NOT EXISTS conscious_state (
387
+ agent_id TEXT PRIMARY KEY,
388
+ last_write_id TEXT,
389
+ last_recall_context TEXT,
390
+ last_recall_ids TEXT NOT NULL DEFAULT '[]',
391
+ last_activity_at TEXT NOT NULL DEFAULT (datetime('now')),
392
+ write_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
393
+ recall_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
394
+ execution_state TEXT,
395
+ checkpoint_at TEXT,
396
+ last_consolidation_at TEXT,
397
+ last_mini_consolidation_at TEXT,
398
+ consolidation_cycle_count INTEGER NOT NULL DEFAULT 0,
399
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
400
+ )
401
+ `);
402
+
403
+ // Migration: add consolidation_cycle_count if missing (existing DBs)
404
+ try {
405
+ this.db.exec(`ALTER TABLE conscious_state ADD COLUMN consolidation_cycle_count INTEGER NOT NULL DEFAULT 0`);
406
+ } catch { /* column already exists */ }
407
+
408
+ // Migration: add memory_type column if missing
409
+ try {
410
+ this.db.prepare('SELECT memory_type FROM engrams LIMIT 0').get();
411
+ } catch {
412
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'unclassified'`);
413
+ }
414
+
415
+ // Migration (0.8 Cluster A): sequence column for story-time / chronology.
416
+ // Nullable — existing engrams stay NULL. Partial index keeps the cost low.
417
+ try {
418
+ this.db.prepare('SELECT sequence FROM engrams LIMIT 0').get();
419
+ } catch {
420
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN sequence INTEGER`);
421
+ this.db.exec(
422
+ `CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence
423
+ ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL`
424
+ );
425
+ }
426
+
427
+ // Migration (0.8 Cluster A): references_json column for typed cross-record
428
+ // links. Wired through HTTP in Cluster D — schema slot only here.
429
+ try {
430
+ this.db.prepare('SELECT references_json FROM engrams LIMIT 0').get();
431
+ } catch {
432
+ this.db.exec(`ALTER TABLE engrams ADD COLUMN references_json TEXT`);
433
+ }
434
+ }
435
+
436
+ // --- Engram CRUD ---
437
+
438
+ createEngram(input: EngramCreate): Engram {
439
+ const now = new Date().toISOString();
440
+ const id = randomUUID();
441
+ const embeddingBlob = input.embedding
442
+ ? Buffer.from(new Float32Array(input.embedding).buffer)
443
+ : null;
444
+
445
+ this.db.prepare(`
446
+ INSERT INTO engrams (id, agent_id, concept, content, embedding, confidence, salience,
447
+ access_count, last_accessed, created_at, salience_features, reason_codes, stage, tags, episode_id,
448
+ ttl, memory_class, supersedes, task_status, task_priority, blocked_by, memory_type,
449
+ sequence, references_json, origin_class, writer_session, recipe_id, valid_from, valid_to)
450
+ VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
451
+ `).run(
452
+ id, input.agentId, input.concept, input.content, embeddingBlob,
453
+ input.confidence ?? 0.5,
454
+ input.salience ?? 0.5,
455
+ now, now,
456
+ JSON.stringify(input.salienceFeatures ?? DEFAULT_SALIENCE_FEATURES),
457
+ JSON.stringify(input.reasonCodes ?? []),
458
+ JSON.stringify(input.tags ?? []),
459
+ input.episodeId ?? null,
460
+ input.ttl ?? null,
461
+ input.memoryClass ?? 'working',
462
+ input.supersedes ?? null,
463
+ input.taskStatus ?? null,
464
+ input.taskPriority ?? null,
465
+ input.blockedBy ?? null,
466
+ input.memoryType ?? 'unclassified',
467
+ input.sequence ?? null,
468
+ input.references && input.references.length > 0
469
+ ? JSON.stringify(input.references) : null,
470
+ input.originClass ?? null,
471
+ input.writerSession ?? null,
472
+ input.recipeId ?? null,
473
+ input.validFrom ?? null,
474
+ input.validTo ?? null,
475
+ );
476
+
477
+ // Add to slim cache (skip if not yet populated — first slim fetch will load it)
478
+ if (this.slimCachePopulated) {
479
+ this.cacheAdd({
480
+ id,
481
+ agentId: input.agentId,
482
+ concept: input.concept,
483
+ embedding: input.embedding ?? null,
484
+ stage: 'active',
485
+ retracted: false,
486
+ });
487
+ }
488
+
489
+ return this.getEngram(id)!;
490
+ }
491
+
492
+ getEngram(id: string): Engram | null {
493
+ const row = this.db.prepare('SELECT * FROM engrams WHERE id = ?').get(id) as any;
494
+ return row ? this.rowToEngram(row) : null;
495
+ }
496
+
497
+ getEngramsByAgent(agentId: string, stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
498
+ let query = 'SELECT * FROM engrams WHERE agent_id = ?';
499
+ const params: any[] = [agentId];
500
+
501
+ if (stage) {
502
+ query += ' AND stage = ?';
503
+ params.push(stage);
504
+ }
505
+ if (!includeRetracted) {
506
+ query += ' AND retracted = 0';
507
+ }
508
+
509
+ return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
510
+ }
511
+
512
+ /**
513
+ * Slim variant that returns only (id, concept, embedding) — the minimum needed
514
+ * for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
515
+ * survival check). Avoids materializing the content blob, tag JSON, salience
516
+ * features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
517
+ *
518
+ * Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
519
+ * engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
520
+ * of that is row materialization of fields we don't read in the filter pass.
521
+ */
522
+ getEngramsByAgentSlim(
523
+ agentId: string,
524
+ stage?: EngramStage,
525
+ includeRetracted: boolean = false
526
+ ): Array<{ id: string; concept: string; embedding: number[] | null }> {
527
+ if (this.slimCacheEnabled) {
528
+ this.ensureSlimCachePopulated();
529
+ const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
530
+ for (const entry of this.slimCache.values()) {
531
+ if (entry.agentId !== agentId) continue;
532
+ if (stage && entry.stage !== stage) continue;
533
+ if (!includeRetracted && entry.retracted) continue;
534
+ result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
535
+ }
536
+ return result;
537
+ }
538
+ // Cache disabled — fall back to direct SQL
539
+ let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
540
+ const params: any[] = [agentId];
541
+
542
+ if (stage) {
543
+ query += ' AND stage = ?';
544
+ params.push(stage);
545
+ }
546
+ if (!includeRetracted) {
547
+ query += ' AND retracted = 0';
548
+ }
549
+
550
+ return (this.db.prepare(query).all(...params) as any[]).map(r => ({
551
+ id: r.id as string,
552
+ concept: r.concept as string,
553
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
554
+ }));
555
+ }
556
+
557
+ /** Slim variant for multi-agent (workspace-scoped) pre-filter. */
558
+ getEngramsByAgentsSlim(
559
+ agentIds: string[],
560
+ stage?: EngramStage,
561
+ includeRetracted: boolean = false
562
+ ): Array<{ id: string; concept: string; embedding: number[] | null }> {
563
+ if (agentIds.length === 0) return [];
564
+ if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
565
+
566
+ if (this.slimCacheEnabled) {
567
+ this.ensureSlimCachePopulated();
568
+ const agentSet = new Set(agentIds);
569
+ const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
570
+ for (const entry of this.slimCache.values()) {
571
+ if (!agentSet.has(entry.agentId)) continue;
572
+ if (stage && entry.stage !== stage) continue;
573
+ if (!includeRetracted && entry.retracted) continue;
574
+ result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
575
+ }
576
+ return result;
577
+ }
578
+
579
+ const placeholders = agentIds.map(() => '?').join(',');
580
+ let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
581
+ const params: any[] = [...agentIds];
582
+
583
+ if (stage) {
584
+ query += ' AND stage = ?';
585
+ params.push(stage);
586
+ }
587
+ if (!includeRetracted) {
588
+ query += ' AND retracted = 0';
589
+ }
590
+
591
+ return (this.db.prepare(query).all(...params) as any[]).map(r => ({
592
+ id: r.id as string,
593
+ concept: r.concept as string,
594
+ embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
595
+ }));
596
+ }
597
+
598
+ /**
599
+ * Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
600
+ * only the survivors that need deep scoring. Chunks IN-clause queries to stay
601
+ * under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
602
+ */
603
+ getEngramsByIds(ids: string[]): Engram[] {
604
+ if (ids.length === 0) return [];
605
+ const CHUNK = 800;
606
+ const result: Engram[] = [];
607
+ for (let i = 0; i < ids.length; i += CHUNK) {
608
+ const chunk = ids.slice(i, i + CHUNK);
609
+ const placeholders = chunk.map(() => '?').join(',');
610
+ const rows = this.db.prepare(
611
+ `SELECT * FROM engrams WHERE id IN (${placeholders})`
612
+ ).all(...chunk) as any[];
613
+ for (const r of rows) result.push(this.rowToEngram(r));
614
+ }
615
+ return result;
616
+ }
617
+
618
+ /**
619
+ * Get engrams across multiple agents (workspace-scoped recall).
620
+ * Used when workspace mode is enabled for hive memory sharing.
621
+ */
622
+ getEngramsByAgents(agentIds: string[], stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
623
+ if (agentIds.length === 0) return [];
624
+ if (agentIds.length === 1) return this.getEngramsByAgent(agentIds[0], stage, includeRetracted);
625
+
626
+ const placeholders = agentIds.map(() => '?').join(',');
627
+ let query = `SELECT * FROM engrams WHERE agent_id IN (${placeholders})`;
628
+ const params: any[] = [...agentIds];
629
+
630
+ if (stage) {
631
+ query += ' AND stage = ?';
632
+ params.push(stage);
633
+ }
634
+ if (!includeRetracted) {
635
+ query += ' AND retracted = 0';
636
+ }
637
+
638
+ return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
639
+ }
640
+
641
+ /**
642
+ * Tags of the most-accessed engrams, bounded IN SQL.
643
+ *
644
+ * Exists because the recall path's pronoun branch used to call
645
+ * `getEngramsByAgents(agentIds, 'active')` — hydrating EVERY active engram
646
+ * into JS, embeddings and all — then sort by accessCount in memory, keep the
647
+ * top ten, and extract five tag words. Measured 2026-10-08 on an
648
+ * 11k-retrievable store: 245ms mean / 280ms median whenever it ran, which was
649
+ * 2.1% of real prompts (the words that trigger it include "it", "this",
650
+ * "that" and "there") against 0.2% of benchmark probes — so it cost real
651
+ * users ~62% of a recall and the benchmark never saw it. See
652
+ * docs/recall-latency.md.
653
+ *
654
+ * ONE QUERY PER AGENT, deliberately. `agent_id IN (...)` makes SQLite abandon
655
+ * the ordered index walk and sort the whole partition in a temp B-tree:
656
+ * measured on the 11k-retrievable store, the IN form costs 86ms against
657
+ * **0.03ms** for the equality form, because only the latter can walk
658
+ * `idx_engrams_access` and stop at LIMIT. Agent-scoped recall passes a single
659
+ * agent and hits the fast path either way; workspace-scoped recall would have
660
+ * paid the 86ms on every pronoun query. The global top N is always contained
661
+ * in the union of the per-agent top Ns, so merging in JS is exact, not an
662
+ * approximation.
663
+ *
664
+ * Ordering is `access_count DESC, id`. The `id` tiebreak is deliberate: the
665
+ * old JS sort broke ties on SQLite's scan order, which is arbitrary, so this
666
+ * is both cheaper and more deterministic.
667
+ */
668
+ getTopAccessedTags(agentIds: string[], topN: number, stage: EngramStage = 'active'): string[][] {
669
+ if (agentIds.length === 0 || topN <= 0) return [];
670
+ const stmt = this.db.prepare(`
671
+ SELECT tags, access_count AS accessCount, id FROM engrams
672
+ WHERE agent_id = ? AND stage = ? AND retracted = 0
673
+ ORDER BY access_count DESC, id
674
+ LIMIT ?
675
+ `);
676
+ const rows: { tags: string; accessCount: number; id: string }[] = [];
677
+ for (const agentId of agentIds) {
678
+ rows.push(...(stmt.all(agentId, stage, topN) as { tags: string; accessCount: number; id: string }[]));
679
+ }
680
+ if (agentIds.length > 1) {
681
+ rows.sort((a, b) => (b.accessCount - a.accessCount) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
682
+ }
683
+ return rows.slice(0, topN).map(r => {
684
+ try { return JSON.parse(r.tags) as string[]; } catch { return []; }
685
+ });
686
+ }
687
+
688
+ /**
689
+ * BM25 search across multiple agents (workspace-scoped).
690
+ */
691
+ searchBM25WithRankMultiAgent(agentIds: string[], query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
692
+ if (agentIds.length === 0) return [];
693
+ if (agentIds.length === 1) return this.searchBM25WithRank(agentIds[0], query, limit);
694
+
695
+ const sanitized = query
696
+ .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
697
+ .split(/\s+/)
698
+ .filter(w => w.length > 1)
699
+ .map(w => `"${w}"`)
700
+ .join(' OR ');
701
+
702
+ if (!sanitized) return [];
703
+
704
+ // CTE prefilter — see searchBM25WithRank for rationale (567× speedup verified).
705
+ try {
706
+ const placeholders = agentIds.map(() => '?').join(',');
707
+ const innerLimit = Math.max(limit * 5, 50);
708
+ const rows = this.db.prepare(`
709
+ WITH top_fts AS (
710
+ SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
711
+ )
712
+ SELECT e.*, top_fts.rank FROM top_fts
713
+ JOIN engrams e ON e.rowid = top_fts.rowid
714
+ WHERE e.agent_id IN (${placeholders}) AND e.retracted = 0
715
+ ORDER BY top_fts.rank
716
+ LIMIT ?
717
+ `).all(sanitized, innerLimit, ...agentIds, limit) as any[];
718
+
719
+ return rows.map(r => ({
720
+ engram: this.rowToEngram(r),
721
+ bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
722
+ }));
723
+ } catch {
724
+ return [];
725
+ }
726
+ }
727
+
728
+ /**
729
+ * Get all distinct agent IDs that share a workspace (requires coord_agents table).
730
+ * Returns just the queried agentId if coordination tables don't exist.
731
+ */
732
+ getWorkspaceAgentIds(agentId: string, workspace: string): string[] {
733
+ try {
734
+ // Return agent names (not UUIDs) — engrams.agent_id uses name strings
735
+ const rows = this.db.prepare(
736
+ `SELECT DISTINCT name FROM coord_agents WHERE workspace = ? AND status != 'dead'`
737
+ ).all(workspace) as Array<{ name: string }>;
738
+ const names = rows.map(r => r.name);
739
+ // Ensure the querying agent is always included
740
+ if (!names.includes(agentId)) names.push(agentId);
741
+ return names;
742
+ } catch {
743
+ // No coordination tables — fall back to single agent
744
+ return [agentId];
745
+ }
746
+ }
747
+
748
+ /**
749
+ * Touch an engram: increment access count, update last_accessed, and
750
+ * nudge confidence upward. Each retrieval is weak evidence the memory
751
+ * is useful — bounded so only explicit feedback can push confidence
752
+ * above 0.85. Diminishing returns: first accesses matter most.
753
+ *
754
+ * Boost: +0.02 per access, scaled by 1/sqrt(accessCount+1), capped at 0.85.
755
+ */
756
+ touchEngram(id: string): void {
757
+ this.db.prepare(`
758
+ UPDATE engrams
759
+ SET access_count = access_count + 1,
760
+ last_accessed = ?,
761
+ confidence = MIN(0.85, confidence + 0.02 / (1.0 + sqrt(access_count)))
762
+ WHERE id = ?
763
+ `).run(new Date().toISOString(), id);
764
+ }
765
+
766
+ /**
767
+ * Move an engram to a new stage, recording WHEN. The timestamp is written here
768
+ * rather than at the six consolidation call sites so a future transition cannot
769
+ * forget it; see `Engram.stageChangedAt` for why the column exists.
770
+ */
771
+ updateStage(id: string, stage: EngramStage): void {
772
+ this.db.prepare('UPDATE engrams SET stage = ?, stage_changed_at = ? WHERE id = ?')
773
+ .run(stage, new Date().toISOString(), id);
774
+ this.cacheUpdateStage(id, stage);
775
+ }
776
+
777
+ /**
778
+ * Replace an engram's content. Used by the fade phase of consolidation
779
+ * (Paper 1: storage degradation) to coarsen un-recalled memories without
780
+ * deleting them — concept + tags stay intact for tag-based retrieval, but
781
+ * BM25 content surface area shrinks.
782
+ */
783
+ updateContent(id: string, content: string): void {
784
+ this.db.prepare('UPDATE engrams SET content = ? WHERE id = ?').run(content, id);
785
+ }
786
+
787
+ updateConfidence(id: string, confidence: number): void {
788
+ this.db.prepare('UPDATE engrams SET confidence = ? WHERE id = ?').run(
789
+ Math.max(0, Math.min(1, confidence)), id
790
+ );
791
+ }
792
+
793
+ updateEmbedding(id: string, embedding: number[], modelId?: string): void {
794
+ const blob = Buffer.from(new Float32Array(embedding).buffer);
795
+ if (modelId) {
796
+ this.db.prepare('UPDATE engrams SET embedding = ?, embedding_model = ? WHERE id = ?').run(blob, modelId, id);
797
+ } else {
798
+ this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
799
+ }
800
+ this.cacheUpdateEmbedding(id, embedding);
801
+ }
802
+
803
+ retractEngram(id: string, retractedBy: string | null): void {
804
+ this.db.prepare(`
805
+ UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
806
+ `).run(retractedBy, new Date().toISOString(), id);
807
+ this.cacheRetract(id);
808
+ }
809
+
810
+ /**
811
+ * Delete an engram and everything that points at it.
812
+ *
813
+ * `associations` cascades (its FKs declare ON DELETE CASCADE), but
814
+ * `retrieval_feedback`, `entity_mentions` and `staging_events` carry a bare
815
+ * `engram_id TEXT` with no foreign key, so deleting an engram used to strand their
816
+ * rows. Measured on the live store 2026-10-12: **571 orphaned retrieval_feedback
817
+ * rows and 3 orphaned entity_mentions**, present in the pre-purge backup too, so
818
+ * they accumulated over time rather than in one incident. Inert for recall, but
819
+ * they corrupt any join that assumes referential integrity — including the
820
+ * feedback-weighted scoring channel's own counts.
821
+ *
822
+ * One transaction, so a crash cannot leave the half-deleted state this is fixing.
823
+ * `activation_events` is deliberately NOT touched: its `engram_ids` is a JSON array
824
+ * describing a recall that really happened, and editing history to remove one id
825
+ * would falsify the event rather than tidy it.
826
+ */
827
+ deleteEngram(id: string): void {
828
+ this.db.transaction(() => {
829
+ this.db.prepare('DELETE FROM retrieval_feedback WHERE engram_id = ?').run(id);
830
+ this.db.prepare('DELETE FROM entity_mentions WHERE engram_id = ?').run(id);
831
+ this.db.prepare('DELETE FROM staging_events WHERE engram_id = ?').run(id);
832
+ this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
833
+ })();
834
+ this.cacheRemove(id);
835
+ }
836
+
837
+ /**
838
+ * Time warp — shift all timestamps backward by ms milliseconds.
839
+ * Used for testing time-dependent behavior (decay, forgetting).
840
+ * Returns count of records shifted.
841
+ */
842
+ timeWarp(agentId: string, ms: number): number {
843
+ let count = 0;
844
+ const shiftSec = Math.round(ms / 1000);
845
+ // Shift engram timestamps
846
+ const r1 = this.db.prepare(`
847
+ UPDATE engrams SET
848
+ created_at = datetime(created_at, '-${shiftSec} seconds'),
849
+ last_accessed = datetime(last_accessed, '-${shiftSec} seconds')
850
+ WHERE agent_id = ?
851
+ `).run(agentId);
852
+ count += r1.changes;
853
+ // Shift association timestamps
854
+ const r2 = this.db.prepare(`
855
+ UPDATE associations SET
856
+ created_at = datetime(created_at, '-${shiftSec} seconds'),
857
+ last_activated = datetime(last_activated, '-${shiftSec} seconds')
858
+ WHERE from_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
859
+ OR to_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
860
+ `).run(agentId, agentId);
861
+ count += r2.changes;
862
+ return count;
863
+ }
864
+
865
+ // --- Full-text search (BM25) ---
866
+
867
+ searchBM25(agentId: string, query: string, limit: number = 10): Engram[] {
868
+ return this.searchBM25WithRank(agentId, query, limit).map(r => r.engram);
869
+ }
870
+
871
+ /**
872
+ * Vector similarity top-K search. SQLite has no native vector index, so we
873
+ * iterate over the slim cache, compute cosine similarity in-process, and
874
+ * return the top-K nearest neighbors. PGlite uses pgvector + ivfflat for
875
+ * the same operation in O(log N).
876
+ *
877
+ * Returns engrams with their cosine distance (0 = identical, 2 = opposite)
878
+ * to match the PGlite contract. Activation engine consumes `1 - distance`
879
+ * as the cosine similarity score.
880
+ */
881
+ searchByVector(
882
+ agentId: string,
883
+ vec: number[],
884
+ limit: number = 10,
885
+ ): Array<{ engram: Engram; distance: number }> {
886
+ if (!vec || vec.length === 0) return [];
887
+ // Include both 'active' and 'fading' — faded engrams retain their embedding
888
+ // (Paper 1: storage degradation; only content is trimmed) so they should still
889
+ // participate in vector recall. Concept + tags + truncated content remain in BM25.
890
+ const activeSlim = this.getEngramsByAgentSlim(agentId, 'active', false);
891
+ const fadingSlim = this.getEngramsByAgentSlim(agentId, 'fading', false);
892
+ const slim = activeSlim.concat(fadingSlim);
893
+ const scored: Array<{ id: string; sim: number }> = [];
894
+ for (const e of slim) {
895
+ if (!e.embedding) continue;
896
+ const sim = cosineSimilarity(vec, e.embedding);
897
+ scored.push({ id: e.id, sim });
898
+ }
899
+ scored.sort((a, b) => b.sim - a.sim);
900
+ const topIds = scored.slice(0, limit).map(s => s.id);
901
+ if (topIds.length === 0) return [];
902
+ const engrams = this.getEngramsByIds(topIds);
903
+ const engramMap = new Map(engrams.map(e => [e.id, e]));
904
+ return scored
905
+ .slice(0, limit)
906
+ .map(s => ({ engram: engramMap.get(s.id), sim: s.sim }))
907
+ .filter((x): x is { engram: Engram; sim: number } => x.engram !== undefined)
908
+ .map(x => ({ engram: x.engram, distance: 1 - x.sim }));
909
+ }
910
+
911
+ /**
912
+ * BM25 search returning rank scores alongside engrams.
913
+ * FTS5 rank is negative (lower = better match).
914
+ * We normalize to 0-1 where higher = better.
915
+ */
916
+ searchBM25WithRank(agentId: string, query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
917
+ // Sanitize query for FTS5: quote each word to prevent column name interpretation
918
+ const sanitized = query
919
+ .replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
920
+ .split(/\s+/)
921
+ .filter(w => w.length > 1)
922
+ .map(w => `"${w}"`)
923
+ .join(' OR ');
924
+
925
+ if (!sanitized) return [];
926
+
927
+ // CTE prefilter: force FTS5 to apply LIMIT before joining engrams.
928
+ //
929
+ // Why: the obvious query (JOIN engrams_fts ON rowid + WHERE MATCH + ORDER BY rank LIMIT N)
930
+ // makes SQLite's planner materialize ALL matching FTS rows joined with engrams
931
+ // before applying LIMIT. With wide OR queries on a 17K-engram index, that's
932
+ // thousands of row materializations including 1.5KB embedding blobs — measured
933
+ // at 3682ms for a 5-term OR query.
934
+ //
935
+ // The CTE forces FTS5 to LIMIT first (sub-ms), then join only the top-K rowids.
936
+ // Same query plan, 567× faster (3682ms → 6ms verified on 17K engrams).
937
+ //
938
+ // The inner LIMIT (limit * 5) over-fetches because the agent_id + retracted
939
+ // filter is applied AFTER the CTE. limit*5 gives enough headroom that filtered
940
+ // results still satisfy the outer LIMIT for typical workloads (single agent
941
+ // dominant, low retracted rate).
942
+ try {
943
+ const innerLimit = Math.max(limit * 5, 50);
944
+ const rows = this.db.prepare(`
945
+ WITH top_fts AS (
946
+ SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
947
+ )
948
+ SELECT e.*, top_fts.rank FROM top_fts
949
+ JOIN engrams e ON e.rowid = top_fts.rowid
950
+ WHERE e.agent_id = ? AND e.retracted = 0
951
+ ORDER BY top_fts.rank
952
+ LIMIT ?
953
+ `).all(sanitized, innerLimit, agentId, limit) as any[];
954
+
955
+ return rows.map(r => ({
956
+ engram: this.rowToEngram(r),
957
+ // Normalize: rank is negative, more negative = better match.
958
+ // |rank| / (1 + |rank|) gives 0-1 where higher = better.
959
+ bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
960
+ }));
961
+ } catch {
962
+ return [];
963
+ }
964
+ }
965
+
966
+ // --- Diagnostic search (deterministic, not cognitive) ---
967
+
968
+ search(query: SearchQuery): Engram[] {
969
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ?';
970
+ const params: any[] = [query.agentId];
971
+
972
+ if (query.text) {
973
+ sql += ' AND (content LIKE ? OR concept LIKE ?)';
974
+ params.push(`%${query.text}%`, `%${query.text}%`);
975
+ }
976
+ if (query.concept) {
977
+ sql += ' AND concept = ?';
978
+ params.push(query.concept);
979
+ }
980
+ if (query.stage) {
981
+ sql += ' AND stage = ?';
982
+ params.push(query.stage);
983
+ }
984
+ if (query.retracted !== undefined) {
985
+ sql += ' AND retracted = ?';
986
+ params.push(query.retracted ? 1 : 0);
987
+ }
988
+
989
+ // ── Tag filters (0.8 Cluster B) ──
990
+ // AND-set: merge legacy `tags` + new `tagsAll`. Both apply (intersection).
991
+ const allTags = [...(query.tags ?? []), ...(query.tagsAll ?? [])];
992
+ if (allTags.length > 0) {
993
+ for (const tag of allTags) {
994
+ sql += ' AND tags LIKE ?';
995
+ params.push(`%"${tag}"%`);
996
+ }
997
+ }
998
+ // OR-set: at least one must match.
999
+ if (query.tagsAny && query.tagsAny.length > 0) {
1000
+ const ors = query.tagsAny.map(() => 'tags LIKE ?').join(' OR ');
1001
+ sql += ` AND (${ors})`;
1002
+ for (const tag of query.tagsAny) params.push(`%"${tag}"%`);
1003
+ }
1004
+ // NOT-set: none may match.
1005
+ if (query.tagsNone && query.tagsNone.length > 0) {
1006
+ const ors = query.tagsNone.map(() => 'tags LIKE ?').join(' OR ');
1007
+ sql += ` AND NOT (${ors})`;
1008
+ for (const tag of query.tagsNone) params.push(`%"${tag}"%`);
1009
+ }
1010
+
1011
+ // ── Sort (0.8 Cluster B) ──
1012
+ // Default preserves legacy behavior (last_accessed DESC). Map TS camelCase
1013
+ // to SQL snake_case. `sequence` puts NULLs last via (col IS NULL) trick.
1014
+ const sortCol = ({
1015
+ createdAt: 'created_at',
1016
+ sequence: 'sequence',
1017
+ salience: 'salience',
1018
+ confidence: 'confidence',
1019
+ lastAccessed: 'last_accessed',
1020
+ } as const)[query.sortBy ?? 'lastAccessed'];
1021
+ const dir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
1022
+ if (query.sortBy === 'sequence') {
1023
+ // NULLs last regardless of direction so engrams without a story-time
1024
+ // value don't shuffle into the middle of sequenced results.
1025
+ sql += ` ORDER BY (sequence IS NULL), sequence ${dir}`;
1026
+ } else {
1027
+ sql += ` ORDER BY ${sortCol} ${dir}`;
1028
+ }
1029
+
1030
+ sql += ` LIMIT ? OFFSET ?`;
1031
+ params.push(query.limit ?? 50, query.offset ?? 0);
1032
+
1033
+ return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
1034
+ }
1035
+
1036
+ /**
1037
+ * Get the most recently created engram for an agent (for temporal adjacency edges).
1038
+ */
1039
+ getLatestEngram(agentId: string, excludeId?: string): Engram | null {
1040
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0';
1041
+ const params: any[] = [agentId];
1042
+ if (excludeId) {
1043
+ sql += ' AND id != ?';
1044
+ params.push(excludeId);
1045
+ }
1046
+ // FIX 2026-08-23: created_at is millisecond-resolution ISO text, so two engrams
1047
+ // written in the same millisecond TIE and "most recent" became whichever row the
1048
+ // engine happened to return. Surfaced on Linux (same-ms writes) while passing on
1049
+ // Windows, whose coarser timer separated them. rowid is monotonic with insert
1050
+ // order, so it is both deterministic AND chronologically correct as a tiebreak.
1051
+ sql += ' ORDER BY created_at DESC, rowid DESC LIMIT 1';
1052
+ const row = this.db.prepare(sql).get(...params) as any;
1053
+ return row ? this.rowToEngram(row) : null;
1054
+ }
1055
+
1056
+ // --- Task management ---
1057
+
1058
+ updateTaskStatus(id: string, status: TaskStatus): void {
1059
+ this.db.prepare('UPDATE engrams SET task_status = ? WHERE id = ?').run(status, id);
1060
+ }
1061
+
1062
+ updateTaskPriority(id: string, priority: TaskPriority): void {
1063
+ this.db.prepare('UPDATE engrams SET task_priority = ? WHERE id = ?').run(priority, id);
1064
+ }
1065
+
1066
+ updateBlockedBy(id: string, blockedBy: string | null): void {
1067
+ this.db.prepare('UPDATE engrams SET blocked_by = ?, task_status = ? WHERE id = ?')
1068
+ .run(blockedBy, blockedBy ? 'blocked' : 'open', id);
1069
+ }
1070
+
1071
+ /**
1072
+ * Get tasks for an agent, optionally filtered by status.
1073
+ * Results ordered by priority (urgent > high > medium > low), then creation date.
1074
+ */
1075
+ getTasks(agentId: string, status?: TaskStatus): Engram[] {
1076
+ let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND task_status IS NOT NULL AND retracted = 0';
1077
+ const params: any[] = [agentId];
1078
+ if (status) {
1079
+ sql += ' AND task_status = ?';
1080
+ params.push(status);
1081
+ }
1082
+ sql += ` ORDER BY
1083
+ CASE task_priority
1084
+ WHEN 'urgent' THEN 0
1085
+ WHEN 'high' THEN 1
1086
+ WHEN 'medium' THEN 2
1087
+ WHEN 'low' THEN 3
1088
+ ELSE 4
1089
+ END,
1090
+ created_at DESC`;
1091
+ return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
1092
+ }
1093
+
1094
+ /**
1095
+ * Get the next actionable task — highest priority that's not blocked or done.
1096
+ */
1097
+ getNextTask(agentId: string): Engram | null {
1098
+ const row = this.db.prepare(`
1099
+ SELECT * FROM engrams
1100
+ WHERE agent_id = ? AND task_status IN ('open', 'in_progress') AND retracted = 0
1101
+ ORDER BY
1102
+ CASE task_status WHEN 'in_progress' THEN 0 ELSE 1 END,
1103
+ CASE task_priority
1104
+ WHEN 'urgent' THEN 0
1105
+ WHEN 'high' THEN 1
1106
+ WHEN 'medium' THEN 2
1107
+ WHEN 'low' THEN 3
1108
+ ELSE 4
1109
+ END,
1110
+ created_at ASC
1111
+ LIMIT 1
1112
+ `).get(agentId) as any;
1113
+ return row ? this.rowToEngram(row) : null;
1114
+ }
1115
+
1116
+ // --- Supersession ---
1117
+
1118
+ /**
1119
+ * Mark an engram as superseded by another.
1120
+ * The old memory stays in the DB (historical) but gets down-ranked in recall.
1121
+ */
1122
+ supersedeEngram(oldId: string, newId: string): void {
1123
+ this.db.prepare('UPDATE engrams SET superseded_by = ? WHERE id = ?').run(newId, oldId);
1124
+ this.db.prepare('UPDATE engrams SET supersedes = ? WHERE id = ?').run(oldId, newId);
1125
+ }
1126
+
1127
+ /**
1128
+ * Find the most recent active engram matching `concept` (case-insensitive,
1129
+ * trimmed) and all of `requiredTags`. Excludes superseded and retracted
1130
+ * engrams. Used by /memory/supersede Form B (0.8 Cluster D) and by
1131
+ * /memory/write's references[] resolution.
1132
+ *
1133
+ * Returns null if no match.
1134
+ */
1135
+ findActiveMatchByConcept(
1136
+ agentId: string,
1137
+ concept: string,
1138
+ requiredTags?: string[],
1139
+ ): Engram | null {
1140
+ let sql = `SELECT * FROM engrams
1141
+ WHERE agent_id = ?
1142
+ AND LOWER(TRIM(concept)) = LOWER(TRIM(?))
1143
+ AND stage = 'active'
1144
+ AND retracted = 0
1145
+ AND superseded_by IS NULL`;
1146
+ const params: any[] = [agentId, concept];
1147
+ if (requiredTags && requiredTags.length > 0) {
1148
+ for (const tag of requiredTags) {
1149
+ sql += ' AND tags LIKE ?';
1150
+ params.push(`%"${tag}"%`);
1151
+ }
1152
+ }
1153
+ // FIX 2026-08-23: created_at is millisecond-resolution ISO text, so two engrams
1154
+ // written in the same millisecond TIE and "most recent" became whichever row the
1155
+ // engine happened to return. Surfaced on Linux (same-ms writes) while passing on
1156
+ // Windows, whose coarser timer separated them. rowid is monotonic with insert
1157
+ // order, so it is both deterministic AND chronologically correct as a tiebreak.
1158
+ sql += ' ORDER BY created_at DESC, rowid DESC LIMIT 1';
1159
+ const row = this.db.prepare(sql).get(...params) as any;
1160
+ return row ? this.rowToEngram(row) : null;
1161
+ }
1162
+
1163
+ /**
1164
+ * Check if an engram has been superseded.
1165
+ */
1166
+ isSuperseded(id: string): boolean {
1167
+ const row = this.db.prepare('SELECT superseded_by FROM engrams WHERE id = ?').get(id) as any;
1168
+ return row?.superseded_by != null;
1169
+ }
1170
+
1171
+ updateMemoryClass(id: string, memoryClass: MemoryClass): void {
1172
+ this.db.prepare('UPDATE engrams SET memory_class = ? WHERE id = ?').run(memoryClass, id);
1173
+ }
1174
+
1175
+ updateTags(id: string, tags: string[]): void {
1176
+ this.db.prepare('UPDATE engrams SET tags = ? WHERE id = ?').run(JSON.stringify(tags), id);
1177
+ }
1178
+
1179
+ // --- Associations ---
1180
+
1181
+ upsertAssociation(
1182
+ fromId: string, toId: string, weight: number,
1183
+ type: AssociationType = 'hebbian', confidence: number = 0.5
1184
+ ): Association {
1185
+ const now = new Date().toISOString();
1186
+ const id = randomUUID();
1187
+
1188
+ this.db.prepare(`
1189
+ INSERT INTO associations (id, from_engram_id, to_engram_id, weight, confidence, type, activation_count, created_at, last_activated)
1190
+ VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)
1191
+ ON CONFLICT(from_engram_id, to_engram_id) DO UPDATE SET
1192
+ weight = ?, confidence = ?, last_activated = ?, activation_count = activation_count + 1
1193
+ `).run(id, fromId, toId, weight, confidence, type, now, now, weight, confidence, now);
1194
+
1195
+ return this.getAssociation(fromId, toId)!;
1196
+ }
1197
+
1198
+ getAssociation(fromId: string, toId: string): Association | null {
1199
+ const row = this.db.prepare(
1200
+ 'SELECT * FROM associations WHERE from_engram_id = ? AND to_engram_id = ?'
1201
+ ).get(fromId, toId) as any;
1202
+ return row ? this.rowToAssociation(row) : null;
1203
+ }
1204
+
1205
+ getAssociationsFor(engramId: string): Association[] {
1206
+ const rows = this.db.prepare(
1207
+ 'SELECT * FROM associations WHERE from_engram_id = ? OR to_engram_id = ?'
1208
+ ).all(engramId, engramId);
1209
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1210
+ }
1211
+
1212
+ /**
1213
+ * Aggregate association stats per engram — count of edges and sum of weights.
1214
+ *
1215
+ * Why: the activation scoring loop only uses `associations.length` (Hebbian gate)
1216
+ * and `sum of weights` (Hebbian mean + centrality). It doesn't read individual
1217
+ * association fields. Returning scalar stats avoids materializing thousands of
1218
+ * Association objects.
1219
+ *
1220
+ * Phase-breakdown spike (2026-05-08, post-0.7.10) showed
1221
+ * `getAssociationsForBatch` over ~300 survivors took 222ms (25% of recall floor).
1222
+ * Stats-only aggregate via a single GROUP BY drops this to ~20ms.
1223
+ *
1224
+ * Graph walk still needs full Association rows, but it operates on top-N
1225
+ * (~30 candidates) — its per-call `getAssociationsFor` is cheap.
1226
+ */
1227
+ getAssociationStatsForBatch(engramIds: string[]): Map<string, { count: number; sumWeight: number }> {
1228
+ const result = new Map<string, { count: number; sumWeight: number }>();
1229
+ if (engramIds.length === 0) return result;
1230
+
1231
+ const CHUNK = 400;
1232
+ for (let i = 0; i < engramIds.length; i += CHUNK) {
1233
+ const chunk = engramIds.slice(i, i + CHUNK);
1234
+ const placeholders = chunk.map(() => '?').join(',');
1235
+ // UNION ALL counts each association once per endpoint that's in the candidate
1236
+ // set — same semantics as the existing getAssociationsForBatch which buckets
1237
+ // associations under both their from and to engram. Self-loops would be
1238
+ // double-counted, but they're rare in practice and the prior code handled them
1239
+ // identically.
1240
+ const rows = this.db.prepare(
1241
+ `SELECT id, SUM(cnt) AS count, SUM(sw) AS sum_weight FROM (
1242
+ SELECT from_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE from_engram_id IN (${placeholders})
1243
+ UNION ALL
1244
+ SELECT to_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE to_engram_id IN (${placeholders})
1245
+ )
1246
+ WHERE id IN (${placeholders})
1247
+ GROUP BY id`
1248
+ ).all(...chunk, ...chunk, ...chunk) as Array<{ id: string; count: number; sum_weight: number }>;
1249
+
1250
+ for (const r of rows) {
1251
+ result.set(r.id, { count: r.count, sumWeight: r.sum_weight });
1252
+ }
1253
+ }
1254
+ // Ensure every requested id has an entry (even zero-edge engrams)
1255
+ for (const id of engramIds) {
1256
+ if (!result.has(id)) result.set(id, { count: 0, sumWeight: 0 });
1257
+ }
1258
+ return result;
1259
+ }
1260
+
1261
+ /**
1262
+ * Batch variant of getAssociationsFor — fetches associations for many engrams
1263
+ * in a single query, returning a Map keyed by engram id.
1264
+ *
1265
+ * Why: per-candidate `getAssociationsFor` calls inside the activation scoring
1266
+ * loop are an N+1. Measured at 1300ms for 10K candidates (sub-ms per call but
1267
+ * accumulating). One IN-clause query reduces this to ~50ms.
1268
+ */
1269
+ getAssociationsForBatch(engramIds: string[]): Map<string, Association[]> {
1270
+ const result = new Map<string, Association[]>();
1271
+ if (engramIds.length === 0) return result;
1272
+
1273
+ // SQLite's default SQLITE_LIMIT_VARIABLE_NUMBER is 999. Chunk to stay safely below.
1274
+ // We bind each id twice (from + to), so chunks of 400 use 800 placeholders.
1275
+ const CHUNK = 400;
1276
+ for (let i = 0; i < engramIds.length; i += CHUNK) {
1277
+ const chunk = engramIds.slice(i, i + CHUNK);
1278
+ const placeholders = chunk.map(() => '?').join(',');
1279
+ const rows = this.db.prepare(
1280
+ `SELECT * FROM associations
1281
+ WHERE from_engram_id IN (${placeholders}) OR to_engram_id IN (${placeholders})`
1282
+ ).all(...chunk, ...chunk) as any[];
1283
+ for (const r of rows) {
1284
+ const a = this.rowToAssociation(r);
1285
+ // Bucket by both endpoints — getAssociationsFor returns either-direction matches.
1286
+ const fromList = result.get(a.fromEngramId) ?? [];
1287
+ fromList.push(a);
1288
+ result.set(a.fromEngramId, fromList);
1289
+ if (a.toEngramId !== a.fromEngramId) {
1290
+ const toList = result.get(a.toEngramId) ?? [];
1291
+ toList.push(a);
1292
+ result.set(a.toEngramId, toList);
1293
+ }
1294
+ }
1295
+ }
1296
+ // Ensure every requested id has an entry (even if empty) so callers can
1297
+ // .get() without null-checking.
1298
+ for (const id of engramIds) {
1299
+ if (!result.has(id)) result.set(id, []);
1300
+ }
1301
+ return result;
1302
+ }
1303
+
1304
+ getOutgoingAssociations(engramId: string): Association[] {
1305
+ const rows = this.db.prepare(
1306
+ 'SELECT * FROM associations WHERE from_engram_id = ?'
1307
+ ).all(engramId);
1308
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1309
+ }
1310
+
1311
+ countAssociationsFor(engramId: string): number {
1312
+ const row = this.db.prepare(
1313
+ 'SELECT COUNT(*) as count FROM associations WHERE from_engram_id = ?'
1314
+ ).get(engramId) as any;
1315
+ return row.count;
1316
+ }
1317
+
1318
+ getWeakestAssociation(engramId: string): Association | null {
1319
+ const row = this.db.prepare(
1320
+ 'SELECT * FROM associations WHERE from_engram_id = ? ORDER BY weight ASC LIMIT 1'
1321
+ ).get(engramId) as any;
1322
+ return row ? this.rowToAssociation(row) : null;
1323
+ }
1324
+
1325
+ deleteAssociation(id: string): void {
1326
+ this.db.prepare('DELETE FROM associations WHERE id = ?').run(id);
1327
+ }
1328
+
1329
+ getAllAssociations(agentId: string): Association[] {
1330
+ const rows = this.db.prepare(`
1331
+ SELECT a.* FROM associations a
1332
+ JOIN engrams e ON a.from_engram_id = e.id
1333
+ WHERE e.agent_id = ?
1334
+ `).all(agentId);
1335
+ return (rows as any[]).map(r => this.rowToAssociation(r));
1336
+ }
1337
+
1338
+ // --- Eviction ---
1339
+
1340
+ getEvictionCandidates(agentId: string, limit: number): Engram[] {
1341
+ // Lowest combined score: low salience + low access + low confidence + oldest
1342
+ const rows = this.db.prepare(`
1343
+ SELECT * FROM engrams
1344
+ WHERE agent_id = ? AND stage = 'active' AND retracted = 0
1345
+ ORDER BY (salience * 0.3 + confidence * 0.3 + (CAST(access_count AS REAL) / (access_count + 5)) * 0.2 +
1346
+ (1.0 / (1.0 + (julianday('now') - julianday(last_accessed)))) * 0.2) ASC
1347
+ LIMIT ?
1348
+ `).all(agentId, limit) as any[];
1349
+ return rows.map(r => this.rowToEngram(r));
1350
+ }
1351
+
1352
+ getActiveCount(agentId: string): number {
1353
+ const row = this.db.prepare(
1354
+ "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'active'"
1355
+ ).get(agentId) as any;
1356
+ return row.count;
1357
+ }
1358
+
1359
+ getStagingCount(agentId: string): number {
1360
+ const row = this.db.prepare(
1361
+ "SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'staging'"
1362
+ ).get(agentId) as any;
1363
+ return row.count;
1364
+ }
1365
+
1366
+ // --- Staging buffer ---
1367
+
1368
+ getExpiredStaging(): Engram[] {
1369
+ const now = Date.now();
1370
+ const rows = this.db.prepare(`
1371
+ SELECT * FROM engrams WHERE stage = 'staging' AND ttl IS NOT NULL
1372
+ `).all() as any[];
1373
+
1374
+ return rows
1375
+ .map(r => this.rowToEngram(r))
1376
+ .filter(e => e.ttl && (e.createdAt.getTime() + e.ttl) < now);
1377
+ }
1378
+
1379
+ // --- Eval event logging ---
1380
+
1381
+ logActivationEvent(event: ActivationEvent): void {
1382
+ this.db.prepare(`
1383
+ INSERT INTO activation_events (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
1384
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1385
+ `).run(
1386
+ event.id, event.agentId, event.timestamp.toISOString(),
1387
+ event.context, event.resultsReturned, event.topScore,
1388
+ event.latencyMs, JSON.stringify(event.engramIds)
1389
+ );
1390
+ }
1391
+
1392
+ logStagingEvent(event: StagingEvent): void {
1393
+ this.db.prepare(`
1394
+ INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
1395
+ VALUES (?, ?, ?, ?, ?, ?)
1396
+ `).run(
1397
+ event.engramId, event.agentId, event.action,
1398
+ event.resonanceScore, event.timestamp.toISOString(), event.ageMs
1399
+ );
1400
+ }
1401
+
1402
+ logRetrievalFeedback(activationEventId: string | null, engramId: string, useful: boolean, context: string): void {
1403
+ this.db.prepare(`
1404
+ INSERT INTO retrieval_feedback (id, activation_event_id, engram_id, useful, context, timestamp)
1405
+ VALUES (?, ?, ?, ?, ?, ?)
1406
+ `).run(randomUUID(), activationEventId, engramId, useful ? 1 : 0, context, new Date().toISOString());
1407
+ }
1408
+
1409
+ // --- Eval metrics queries ---
1410
+
1411
+ getRetrievalPrecision(agentId: string, windowHours: number = 24): number {
1412
+ const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1413
+ const row = this.db.prepare(`
1414
+ SELECT
1415
+ COUNT(CASE WHEN useful = 1 THEN 1 END) as useful_count,
1416
+ COUNT(*) as total_count
1417
+ FROM retrieval_feedback rf
1418
+ LEFT JOIN activation_events ae ON rf.activation_event_id = ae.id
1419
+ JOIN engrams e ON rf.engram_id = e.id
1420
+ WHERE e.agent_id = ? AND rf.timestamp > ?
1421
+ `).get(agentId, since) as any;
1422
+
1423
+ return row.total_count > 0 ? row.useful_count / row.total_count : 0;
1424
+ }
1425
+
1426
+ getStagingMetrics(agentId: string): { promoted: number; discarded: number; expired: number } {
1427
+ const row = this.db.prepare(`
1428
+ SELECT
1429
+ COUNT(CASE WHEN action = 'promoted' THEN 1 END) as promoted,
1430
+ COUNT(CASE WHEN action = 'discarded' THEN 1 END) as discarded,
1431
+ COUNT(CASE WHEN action = 'expired' THEN 1 END) as expired
1432
+ FROM staging_events WHERE agent_id = ?
1433
+ `).get(agentId) as any;
1434
+ return { promoted: row.promoted, discarded: row.discarded, expired: row.expired };
1435
+ }
1436
+
1437
+ getActivationStats(agentId: string, windowHours: number = 24): {
1438
+ count: number; avgLatencyMs: number; p50LatencyMs: number; p90LatencyMs: number; p95LatencyMs: number;
1439
+ } {
1440
+ const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1441
+ const rows = this.db.prepare(`
1442
+ SELECT latency_ms FROM activation_events
1443
+ WHERE agent_id = ? AND timestamp > ?
1444
+ ORDER BY latency_ms ASC
1445
+ `).all(agentId, since) as { latency_ms: number }[];
1446
+
1447
+ if (rows.length === 0) return { count: 0, avgLatencyMs: 0, p50LatencyMs: 0, p90LatencyMs: 0, p95LatencyMs: 0 };
1448
+
1449
+ // 0.14.3: p50/p90 added. The column mixes warm recalls with cold model loads
1450
+ // and stalls (live store: mean 20 s, median 1.6 s, max 38 min), so the mean
1451
+ // is not a usable number; percentiles are. Rows are already sorted ASC.
1452
+ const pct = (q: number) => rows[Math.min(Math.floor(rows.length * q), rows.length - 1)].latency_ms;
1453
+ const total = rows.reduce((s, r) => s + r.latency_ms, 0);
1454
+ return {
1455
+ count: rows.length,
1456
+ avgLatencyMs: total / rows.length,
1457
+ p50LatencyMs: pct(0.5),
1458
+ p90LatencyMs: pct(0.9),
1459
+ p95LatencyMs: pct(0.95),
1460
+ };
1461
+ }
1462
+
1463
+ /**
1464
+ * 0.14.3: feedback rows in the window that are JOINED to an activation event.
1465
+ * Unlinked rows (activation_event_id IS NULL — all 872 rows written before
1466
+ * 0.14.3) are excluded on purpose: this is the recall→use signal, and a row
1467
+ * that cannot be traced to a recall does not measure it.
1468
+ */
1469
+ getLinkedFeedbackStats(agentId: string, windowHours: number = 24 * 7): { total: number; useful: number } {
1470
+ const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
1471
+ const row = this.db.prepare(`
1472
+ SELECT COUNT(*) AS total, COUNT(CASE WHEN rf.useful = 1 THEN 1 END) AS useful
1473
+ FROM retrieval_feedback rf
1474
+ JOIN activation_events ae ON ae.id = rf.activation_event_id
1475
+ WHERE ae.agent_id = ? AND rf.timestamp > ?
1476
+ `).get(agentId, since) as { total: number; useful: number };
1477
+ return { total: row.total, useful: row.useful };
1478
+ }
1479
+
1480
+ getConsolidatedCount(agentId: string): number {
1481
+ const row = this.db.prepare(
1482
+ `SELECT COUNT(*) as cnt FROM engrams WHERE agent_id = ? AND stage = 'consolidated'`
1483
+ ).get(agentId) as any;
1484
+ return row.cnt;
1485
+ }
1486
+
1487
+ // --- Helpers ---
1488
+
1489
+ private rowToEngram(row: any): Engram {
1490
+ return {
1491
+ id: row.id,
1492
+ agentId: row.agent_id,
1493
+ concept: row.concept,
1494
+ content: row.content,
1495
+ embedding: row.embedding
1496
+ ? Array.from(bufferToFloat32Array(row.embedding))
1497
+ : null,
1498
+ confidence: row.confidence,
1499
+ salience: row.salience,
1500
+ accessCount: row.access_count,
1501
+ lastAccessed: new Date(row.last_accessed),
1502
+ createdAt: new Date(row.created_at),
1503
+ salienceFeatures: JSON.parse(row.salience_features || '{}'),
1504
+ reasonCodes: JSON.parse(row.reason_codes || '[]'),
1505
+ stage: row.stage as EngramStage,
1506
+ ttl: row.ttl,
1507
+ retracted: !!row.retracted,
1508
+ retractedBy: row.retracted_by,
1509
+ retractedAt: row.retracted_at ? new Date(row.retracted_at) : null,
1510
+ tags: JSON.parse(row.tags),
1511
+ episodeId: row.episode_id ?? null,
1512
+ memoryClass: (row.memory_class ?? 'working') as MemoryClass,
1513
+ memoryType: (row.memory_type ?? 'unclassified') as MemoryType,
1514
+ supersededBy: row.superseded_by ?? null,
1515
+ supersedes: row.supersedes ?? null,
1516
+ taskStatus: row.task_status ?? null,
1517
+ taskPriority: row.task_priority ?? null,
1518
+ blockedBy: row.blocked_by ?? null,
1519
+ sequence: row.sequence == null ? null : Number(row.sequence),
1520
+ references: row.references_json ? JSON.parse(row.references_json) : null,
1521
+ originClass: row.origin_class ?? null,
1522
+ writerSession: row.writer_session ?? null,
1523
+ recipeId: row.recipe_id ?? null,
1524
+ validFrom: row.valid_from ?? null,
1525
+ validTo: row.valid_to ?? null,
1526
+ stageChangedAt: row.stage_changed_at ?? null,
1527
+ };
1528
+ }
1529
+
1530
+ private rowToAssociation(row: any): Association {
1531
+ return {
1532
+ id: row.id,
1533
+ fromEngramId: row.from_engram_id,
1534
+ toEngramId: row.to_engram_id,
1535
+ weight: row.weight,
1536
+ confidence: row.confidence ?? 0.5,
1537
+ type: row.type as AssociationType,
1538
+ activationCount: row.activation_count ?? 0,
1539
+ createdAt: new Date(row.created_at),
1540
+ lastActivated: new Date(row.last_activated),
1541
+ };
1542
+ }
1543
+
1544
+ // --- Episodes ---
1545
+
1546
+ createEpisode(input: { agentId: string; label: string; embedding?: number[] }): Episode {
1547
+ const now = new Date().toISOString();
1548
+ const id = randomUUID();
1549
+ const embeddingBlob = input.embedding
1550
+ ? Buffer.from(new Float32Array(input.embedding).buffer)
1551
+ : null;
1552
+
1553
+ this.db.prepare(`
1554
+ INSERT INTO episodes (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
1555
+ VALUES (?, ?, ?, ?, 0, ?, ?, ?)
1556
+ `).run(id, input.agentId, input.label, embeddingBlob, now, now, now);
1557
+
1558
+ return this.getEpisode(id)!;
1559
+ }
1560
+
1561
+ getEpisode(id: string): Episode | null {
1562
+ const row = this.db.prepare('SELECT * FROM episodes WHERE id = ?').get(id) as any;
1563
+ return row ? this.rowToEpisode(row) : null;
1564
+ }
1565
+
1566
+ getEpisodesByAgent(agentId: string): Episode[] {
1567
+ const rows = this.db.prepare(
1568
+ 'SELECT * FROM episodes WHERE agent_id = ? ORDER BY end_time DESC'
1569
+ ).all(agentId) as any[];
1570
+ return rows.map(r => this.rowToEpisode(r));
1571
+ }
1572
+
1573
+ getActiveEpisode(agentId: string, windowMs: number = 3600_000): Episode | null {
1574
+ // Find most recent episode that ended within the time window
1575
+ const cutoff = new Date(Date.now() - windowMs).toISOString();
1576
+ const row = this.db.prepare(`
1577
+ SELECT * FROM episodes WHERE agent_id = ? AND end_time > ?
1578
+ ORDER BY end_time DESC LIMIT 1
1579
+ `).get(agentId, cutoff) as any;
1580
+ return row ? this.rowToEpisode(row) : null;
1581
+ }
1582
+
1583
+ addEngramToEpisode(engramId: string, episodeId: string): void {
1584
+ this.db.prepare('UPDATE engrams SET episode_id = ? WHERE id = ?').run(episodeId, engramId);
1585
+ this.db.prepare(`
1586
+ UPDATE episodes SET
1587
+ engram_count = engram_count + 1,
1588
+ end_time = MAX(end_time, ?)
1589
+ WHERE id = ?
1590
+ `).run(new Date().toISOString(), episodeId);
1591
+ }
1592
+
1593
+ getEngramsByEpisode(episodeId: string): Engram[] {
1594
+ const rows = this.db.prepare(
1595
+ 'SELECT * FROM engrams WHERE episode_id = ? AND retracted = 0 ORDER BY created_at ASC'
1596
+ ).all(episodeId) as any[];
1597
+ return rows.map(r => this.rowToEngram(r));
1598
+ }
1599
+
1600
+ updateEpisodeEmbedding(id: string, embedding: number[]): void {
1601
+ const blob = Buffer.from(new Float32Array(embedding).buffer);
1602
+ this.db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(blob, id);
1603
+ }
1604
+
1605
+ getEpisodeCount(agentId: string): number {
1606
+ const row = this.db.prepare(
1607
+ 'SELECT COUNT(*) as cnt FROM episodes WHERE agent_id = ?'
1608
+ ).get(agentId) as any;
1609
+ return row.cnt;
1610
+ }
1611
+
1612
+ private rowToEpisode(row: any): Episode {
1613
+ return {
1614
+ id: row.id,
1615
+ agentId: row.agent_id,
1616
+ label: row.label,
1617
+ embedding: row.embedding
1618
+ ? Array.from(bufferToFloat32Array(row.embedding))
1619
+ : null,
1620
+ engramCount: row.engram_count,
1621
+ startTime: new Date(row.start_time),
1622
+ endTime: new Date(row.end_time),
1623
+ createdAt: new Date(row.created_at),
1624
+ };
1625
+ }
1626
+
1627
+ /**
1628
+ * Find engrams whose tags contain any of the given tag values.
1629
+ * Used for entity-bridge retrieval: given entity tags from top results,
1630
+ * find other engrams mentioning the same entities.
1631
+ */
1632
+ findEngramsByTags(agentId: string, tags: string[], excludeIds?: Set<string>): Engram[] {
1633
+ if (tags.length === 0) return [];
1634
+
1635
+ // Build OR conditions for tag matching
1636
+ const conditions = tags.map(() => 'tags LIKE ?').join(' OR ');
1637
+ const params: any[] = [agentId, ...tags.map(t => `%"${t}"%`)];
1638
+
1639
+ let sql = `SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0 AND (${conditions})`;
1640
+ const rows = this.db.prepare(sql).all(...params) as any[];
1641
+
1642
+ const results = rows.map(r => this.rowToEngram(r));
1643
+ if (excludeIds) {
1644
+ return results.filter(e => !excludeIds.has(e.id));
1645
+ }
1646
+ return results;
1647
+ }
1648
+
1649
+ // --- Checkpointing ---
1650
+
1651
+ updateAutoCheckpointWrite(agentId: string, engramId: string): void {
1652
+ const now = new Date().toISOString();
1653
+ this.db.prepare(`
1654
+ INSERT INTO conscious_state (agent_id, last_write_id, last_activity_at, write_count_since_consolidation, updated_at)
1655
+ VALUES (?, ?, ?, 1, ?)
1656
+ ON CONFLICT(agent_id) DO UPDATE SET
1657
+ last_write_id = excluded.last_write_id,
1658
+ last_activity_at = excluded.last_activity_at,
1659
+ write_count_since_consolidation = write_count_since_consolidation + 1,
1660
+ updated_at = excluded.updated_at
1661
+ `).run(agentId, engramId, now, now);
1662
+ }
1663
+
1664
+ updateAutoCheckpointRecall(agentId: string, context: string, engramIds: string[]): void {
1665
+ const now = new Date().toISOString();
1666
+ this.db.prepare(`
1667
+ INSERT INTO conscious_state (agent_id, last_recall_context, last_recall_ids, last_activity_at, recall_count_since_consolidation, updated_at)
1668
+ VALUES (?, ?, ?, ?, 1, ?)
1669
+ ON CONFLICT(agent_id) DO UPDATE SET
1670
+ last_recall_context = excluded.last_recall_context,
1671
+ last_recall_ids = excluded.last_recall_ids,
1672
+ last_activity_at = excluded.last_activity_at,
1673
+ recall_count_since_consolidation = recall_count_since_consolidation + 1,
1674
+ updated_at = excluded.updated_at
1675
+ `).run(agentId, context, JSON.stringify(engramIds), now, now);
1676
+ }
1677
+
1678
+ touchActivity(agentId: string): void {
1679
+ const now = new Date().toISOString();
1680
+ this.db.prepare(`
1681
+ INSERT INTO conscious_state (agent_id, last_activity_at, updated_at)
1682
+ VALUES (?, ?, ?)
1683
+ ON CONFLICT(agent_id) DO UPDATE SET
1684
+ last_activity_at = excluded.last_activity_at,
1685
+ updated_at = excluded.updated_at
1686
+ `).run(agentId, now, now);
1687
+ }
1688
+
1689
+ saveCheckpoint(agentId: string, state: ConsciousState): void {
1690
+ const now = new Date().toISOString();
1691
+ this.db.prepare(`
1692
+ INSERT INTO conscious_state (agent_id, execution_state, checkpoint_at, last_activity_at, updated_at)
1693
+ VALUES (?, ?, ?, ?, ?)
1694
+ ON CONFLICT(agent_id) DO UPDATE SET
1695
+ execution_state = excluded.execution_state,
1696
+ checkpoint_at = excluded.checkpoint_at,
1697
+ last_activity_at = excluded.last_activity_at,
1698
+ updated_at = excluded.updated_at
1699
+ `).run(agentId, JSON.stringify(state), now, now, now);
1700
+ }
1701
+
1702
+ /** Record entity mentions for an engram (D9 inverted index; idempotent). */
1703
+ recordEntityMentions(engramId: string, agentId: string, entities: string[]): void {
1704
+ if (!entities || entities.length === 0) return;
1705
+ const stmt = this.db.prepare('INSERT OR IGNORE INTO entity_mentions (entity, engram_id, agent_id) VALUES (?, ?, ?)');
1706
+ for (const entity of entities) stmt.run(entity, engramId, agentId);
1707
+ }
1708
+
1709
+ /** Indexed entities whose normalized key:value — or a registered alias — contains the term
1710
+ * (D11 query-side lookup). Alias matches resolve to their target entity. */
1711
+ searchEntities(term: string, limit: number = 8): string[] {
1712
+ const t = term.toLowerCase().replace(/[%_]/g, '');
1713
+ if (!t) return [];
1714
+ const rows = this.db.prepare(`
1715
+ SELECT DISTINCT entity FROM (
1716
+ SELECT entity FROM entity_mentions WHERE entity LIKE ?
1717
+ UNION SELECT entity FROM entity_aliases WHERE alias LIKE ?
1718
+ ) LIMIT ?`).all(`%${t}%`, `%${t}%`, limit) as Array<{ entity: string }>;
1719
+ return rows.map(r => r.entity);
1720
+ }
1721
+
1722
+ /** Engram ids mentioning an entity (alias-resolved), optionally agent-scoped. */
1723
+ getEngramIdsByEntity(entity: string, agentId?: string): string[] {
1724
+ const resolved = (this.db.prepare('SELECT entity FROM entity_aliases WHERE alias = ?')
1725
+ .get(entity.toLowerCase()) as { entity: string } | undefined)?.entity ?? entity.toLowerCase();
1726
+ const rows = agentId
1727
+ ? this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ? AND agent_id = ?').all(resolved, agentId)
1728
+ : this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ?').all(resolved);
1729
+ return (rows as Array<{ engram_id: string }>).map(r => r.engram_id);
1730
+ }
1731
+
1732
+ /** Distinct agent ids present in this store (D3 whoami; includes all stages). */
1733
+ listAgentIds(): string[] {
1734
+ const rows = this.db.prepare('SELECT DISTINCT agent_id FROM engrams ORDER BY agent_id').all() as Array<{ agent_id: string }>;
1735
+ return rows.map(r => r.agent_id);
1736
+ }
1737
+
1738
+ getCheckpoint(agentId: string): CheckpointRow | null {
1739
+ const row = this.db.prepare('SELECT * FROM conscious_state WHERE agent_id = ?').get(agentId) as any;
1740
+ if (!row) return null;
1741
+
1742
+ return {
1743
+ agentId: row.agent_id,
1744
+ auto: {
1745
+ lastWriteId: row.last_write_id ?? null,
1746
+ lastRecallContext: row.last_recall_context ?? null,
1747
+ lastRecallIds: JSON.parse(row.last_recall_ids || '[]'),
1748
+ lastActivityAt: new Date(row.last_activity_at),
1749
+ writeCountSinceConsolidation: row.write_count_since_consolidation,
1750
+ recallCountSinceConsolidation: row.recall_count_since_consolidation,
1751
+ },
1752
+ executionState: row.execution_state ? JSON.parse(row.execution_state) : null,
1753
+ checkpointAt: row.checkpoint_at ? new Date(row.checkpoint_at) : null,
1754
+ lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1755
+ lastMiniConsolidationAt: row.last_mini_consolidation_at ? new Date(row.last_mini_consolidation_at) : null,
1756
+ updatedAt: new Date(row.updated_at),
1757
+ };
1758
+ }
1759
+
1760
+ markConsolidation(agentId: string, mini: boolean): void {
1761
+ const now = new Date().toISOString();
1762
+ if (mini) {
1763
+ this.db.prepare(`
1764
+ UPDATE conscious_state SET last_mini_consolidation_at = ?, updated_at = ? WHERE agent_id = ?
1765
+ `).run(now, now, agentId);
1766
+ } else {
1767
+ this.db.prepare(`
1768
+ UPDATE conscious_state SET
1769
+ last_consolidation_at = ?,
1770
+ last_mini_consolidation_at = ?,
1771
+ write_count_since_consolidation = 0,
1772
+ recall_count_since_consolidation = 0,
1773
+ consolidation_cycle_count = consolidation_cycle_count + 1,
1774
+ updated_at = ?
1775
+ WHERE agent_id = ?
1776
+ `).run(now, now, now, agentId);
1777
+ }
1778
+ }
1779
+
1780
+ getActiveAgents(): Array<{ agentId: string; lastActivityAt: Date; writeCount: number; recallCount: number; lastConsolidationAt: Date | null }> {
1781
+ const rows = this.db.prepare('SELECT * FROM conscious_state').all() as any[];
1782
+ return rows.map(row => ({
1783
+ agentId: row.agent_id,
1784
+ lastActivityAt: new Date(row.last_activity_at),
1785
+ writeCount: row.write_count_since_consolidation,
1786
+ recallCount: row.recall_count_since_consolidation,
1787
+ lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
1788
+ }));
1789
+ }
1790
+
1791
+ getConsolidationCycleCount(agentId: string): number {
1792
+ const row = this.db.prepare(
1793
+ 'SELECT consolidation_cycle_count FROM conscious_state WHERE agent_id = ?',
1794
+ ).get(agentId) as { consolidation_cycle_count: number } | undefined;
1795
+ return row?.consolidation_cycle_count ?? 0;
1796
+ }
1797
+
1798
+ close(): void {
1799
+ this.stopWalCheckpointTimer();
1800
+ this.walCheckpoint();
1801
+ this.db.close();
1802
+ }
1803
+
1804
+ /**
1805
+ * Run `fn` in a SQL transaction. All writes succeed atomically or roll back
1806
+ * on throw. Synchronous (matches better-sqlite3 semantics).
1807
+ *
1808
+ * Used by 0.8 Cluster D's `/memory/supersede` Form B to wrap find-match +
1809
+ * performWrite + supersede + causal-edge + confidence-decay so callers
1810
+ * never observe an intermediate state where new engram exists but old
1811
+ * isn't yet superseded.
1812
+ */
1813
+ transaction<T>(fn: () => T): T {
1814
+ return this.db.transaction(fn)();
1815
+ }
1816
+
1817
+ /**
1818
+ * Async-aware transaction wrapper. `fn` may await — we hold the SQLite
1819
+ * lock across awaits by issuing raw BEGIN/COMMIT/ROLLBACK statements rather
1820
+ * than using better-sqlite3's sync `db.transaction()`.
1821
+ *
1822
+ * Used by Form B (atomic write + supersede) where the body calls
1823
+ * async store methods through the IEngramStore contract.
1824
+ */
1825
+ async withTransaction<T>(fn: () => Promise<T>): Promise<T> {
1826
+ this.db.exec('BEGIN');
1827
+ try {
1828
+ const result = await fn();
1829
+ this.db.exec('COMMIT');
1830
+ return result;
1831
+ } catch (err) {
1832
+ try { this.db.exec('ROLLBACK'); } catch { /* best-effort */ }
1833
+ throw err;
1834
+ }
1835
+ }
1836
+
1837
+ // ============================================================
1838
+ // 0.8 Cluster C — materialized-view + atomic-counter primitives
1839
+ // ============================================================
1840
+
1841
+ /**
1842
+ * For each distinct value of the tag whose key prefix is `tagKeyPrefix`
1843
+ * (e.g. `"character="`), return the most-recent active engram. Used by
1844
+ * NovelForge's "latest emotional state per character", "latest motif
1845
+ * phase per motif", "recent chapter summaries".
1846
+ *
1847
+ * `sortBy` defaults to `createdAt`. With `"sequence"`, engrams without a
1848
+ * sequence are excluded (they have no story-time anchor).
1849
+ */
1850
+ getLatestByTag(opts: {
1851
+ agentId: string;
1852
+ tagKeyPrefix: string; // e.g. "character=", "motif=", "chapter="
1853
+ scopeTagsAll?: string[]; // optional narrowing (e.g. ["topic=motif-use"])
1854
+ retracted?: boolean;
1855
+ sortBy?: 'createdAt' | 'sequence';
1856
+ limit?: number;
1857
+ }): Engram[] {
1858
+ let sql = `SELECT * FROM engrams
1859
+ WHERE agent_id = ?
1860
+ AND retracted = ?
1861
+ AND stage = 'active'
1862
+ AND tags LIKE ?`;
1863
+ const params: any[] = [
1864
+ opts.agentId,
1865
+ opts.retracted ? 1 : 0,
1866
+ `%"${opts.tagKeyPrefix}%`,
1867
+ ];
1868
+ if (opts.scopeTagsAll && opts.scopeTagsAll.length > 0) {
1869
+ for (const t of opts.scopeTagsAll) {
1870
+ sql += ' AND tags LIKE ?';
1871
+ params.push(`%"${t}"%`);
1872
+ }
1873
+ }
1874
+ if (opts.sortBy === 'sequence') {
1875
+ sql += ' AND sequence IS NOT NULL';
1876
+ }
1877
+ sql += ' ORDER BY ' + (opts.sortBy === 'sequence' ? 'sequence DESC, created_at DESC' : 'created_at DESC');
1878
+
1879
+ const rows = this.db.prepare(sql).all(...params) as any[];
1880
+ const engrams = rows.map(r => this.rowToEngram(r));
1881
+
1882
+ // Group by extracted tag value, take first (already sorted newest first).
1883
+ const seen = new Map<string, Engram>();
1884
+ for (const e of engrams) {
1885
+ const value = this.extractTagValue(e.tags, opts.tagKeyPrefix);
1886
+ if (value == null) continue;
1887
+ if (!seen.has(value)) seen.set(value, e);
1888
+ }
1889
+ const out = Array.from(seen.values());
1890
+ return opts.limit ? out.slice(0, opts.limit) : out;
1891
+ }
1892
+
1893
+ /**
1894
+ * Filter engrams by tag-set operators and sort by numeric value extracted
1895
+ * from a tag prefix (e.g. `"weight="` → numeric value of `weight=8`).
1896
+ * Used by NovelForge's "top 40 active promises by weight".
1897
+ *
1898
+ * Missing or unparseable numeric values sort last.
1899
+ */
1900
+ getTopBy(opts: {
1901
+ agentId: string;
1902
+ sortField: string; // tag prefix, e.g. "weight="
1903
+ order: 'asc' | 'desc';
1904
+ filterTagsAll?: string[];
1905
+ filterTagsAny?: string[];
1906
+ filterTagsNone?: string[];
1907
+ limit?: number;
1908
+ retracted?: boolean;
1909
+ }): Engram[] {
1910
+ // Build the base query inline (mirrors search() but adds the topBy sort).
1911
+ let sql = `SELECT * FROM engrams
1912
+ WHERE agent_id = ?
1913
+ AND retracted = ?
1914
+ AND stage = 'active'
1915
+ AND tags LIKE ?`;
1916
+ const params: any[] = [
1917
+ opts.agentId,
1918
+ opts.retracted ? 1 : 0,
1919
+ `%"${opts.sortField}%`, // must have a tag with this prefix
1920
+ ];
1921
+
1922
+ if (opts.filterTagsAll && opts.filterTagsAll.length > 0) {
1923
+ for (const tag of opts.filterTagsAll) {
1924
+ sql += ' AND tags LIKE ?';
1925
+ params.push(`%"${tag}"%`);
1926
+ }
1927
+ }
1928
+ if (opts.filterTagsAny && opts.filterTagsAny.length > 0) {
1929
+ const ors = opts.filterTagsAny.map(() => 'tags LIKE ?').join(' OR ');
1930
+ sql += ` AND (${ors})`;
1931
+ for (const tag of opts.filterTagsAny) params.push(`%"${tag}"%`);
1932
+ }
1933
+ if (opts.filterTagsNone && opts.filterTagsNone.length > 0) {
1934
+ const ors = opts.filterTagsNone.map(() => 'tags LIKE ?').join(' OR ');
1935
+ sql += ` AND NOT (${ors})`;
1936
+ for (const tag of opts.filterTagsNone) params.push(`%"${tag}"%`);
1937
+ }
1938
+
1939
+ const rows = this.db.prepare(sql).all(...params) as any[];
1940
+ const engrams = rows.map(r => this.rowToEngram(r));
1941
+
1942
+ // Extract numeric value from the sortField tag; missing or NaN sorts last.
1943
+ const valued = engrams.map(e => {
1944
+ const raw = this.extractTagValue(e.tags, opts.sortField);
1945
+ const n = raw == null ? NaN : Number(raw);
1946
+ return { e, n };
1947
+ });
1948
+ valued.sort((a, b) => {
1949
+ const aNaN = Number.isNaN(a.n);
1950
+ const bNaN = Number.isNaN(b.n);
1951
+ if (aNaN && bNaN) return 0;
1952
+ if (aNaN) return 1; // NaN last
1953
+ if (bNaN) return -1;
1954
+ return opts.order === 'asc' ? a.n - b.n : b.n - a.n;
1955
+ });
1956
+ const sorted = valued.map(v => v.e);
1957
+ return opts.limit ? sorted.slice(0, opts.limit) : sorted;
1958
+ }
1959
+
1960
+ /**
1961
+ * Compute the effective state of an engram from referenced events.
1962
+ *
1963
+ * - 'superseded' if engram.supersededBy is set
1964
+ * - else: scan engrams whose references_json names this engram with a
1965
+ * terminal relation type (resolves / subverts / abandons). Take the
1966
+ * latest by createdAt. effectiveState = that type.
1967
+ * - else: 'active'
1968
+ *
1969
+ * Returns `null` if target engram not found.
1970
+ */
1971
+ resolveEffectiveState(targetEngramId: string): {
1972
+ engram: Engram;
1973
+ effectiveState: 'active' | 'resolved' | 'subverted' | 'abandoned' | 'superseded';
1974
+ resolvingEvents: Array<{ id: string; type: string; createdAt: string }>;
1975
+ } | null {
1976
+ const target = this.getEngram(targetEngramId);
1977
+ if (!target) return null;
1978
+
1979
+ if (target.supersededBy) {
1980
+ return { engram: target, effectiveState: 'superseded', resolvingEvents: [] };
1981
+ }
1982
+
1983
+ // Find candidate referencing engrams. LIKE-filter first to narrow,
1984
+ // then JSON.parse each candidate's references_json to verify the
1985
+ // matchEngramId and type. Cheap enough at NovelForge's scale.
1986
+ const candidates = this.db.prepare(
1987
+ `SELECT id, references_json, created_at FROM engrams
1988
+ WHERE agent_id = ?
1989
+ AND retracted = 0
1990
+ AND stage = 'active'
1991
+ AND references_json IS NOT NULL
1992
+ AND references_json LIKE ?`
1993
+ ).all(target.agentId, `%"${targetEngramId}"%`) as Array<{
1994
+ id: string;
1995
+ references_json: string;
1996
+ created_at: string;
1997
+ }>;
1998
+
1999
+ const terminalTypes = new Set(['resolves', 'subverts', 'abandons']);
2000
+ const resolvingEvents: Array<{ id: string; type: string; createdAt: string }> = [];
2001
+ for (const c of candidates) {
2002
+ try {
2003
+ const refs = JSON.parse(c.references_json) as Array<{ type: string; matchEngramId?: string }>;
2004
+ for (const ref of refs) {
2005
+ if (ref.matchEngramId === targetEngramId && terminalTypes.has(ref.type)) {
2006
+ resolvingEvents.push({ id: c.id, type: ref.type, createdAt: c.created_at });
2007
+ }
2008
+ }
2009
+ } catch { /* malformed references_json — skip */ }
2010
+ }
2011
+
2012
+ if (resolvingEvents.length === 0) {
2013
+ return { engram: target, effectiveState: 'active', resolvingEvents: [] };
2014
+ }
2015
+
2016
+ // Latest event wins.
2017
+ resolvingEvents.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
2018
+ const latestType = resolvingEvents[0]!.type as 'resolves' | 'subverts' | 'abandons';
2019
+ const effectiveState = (
2020
+ latestType === 'resolves' ? 'resolved'
2021
+ : latestType === 'subverts' ? 'subverted'
2022
+ : 'abandoned'
2023
+ );
2024
+ return { engram: target, effectiveState, resolvingEvents };
2025
+ }
2026
+
2027
+ /**
2028
+ * Atomically allocate the next sequence number for an agent.
2029
+ *
2030
+ * Uses BEGIN IMMEDIATE to serialize concurrent callers — better-sqlite3's
2031
+ * transaction() acquires a RESERVED lock which prevents other writers
2032
+ * from racing. Returns MAX(sequence)+1 for the agent, or 1 if no engrams
2033
+ * have a sequence yet.
2034
+ */
2035
+ allocateNextSequence(agentId: string): number {
2036
+ return this.transaction(() => {
2037
+ const row = this.db.prepare(
2038
+ 'SELECT MAX(sequence) AS max_seq FROM engrams WHERE agent_id = ?'
2039
+ ).get(agentId) as { max_seq: number | null };
2040
+ return (row?.max_seq ?? 0) + 1;
2041
+ });
2042
+ }
2043
+
2044
+ /**
2045
+ * Extract the value following a tag prefix, e.g. `["weight=8"]` + `"weight="`
2046
+ * → `"8"`. Returns null if no tag with that prefix is present.
2047
+ */
2048
+ private extractTagValue(tags: string[], tagKeyPrefix: string): string | null {
2049
+ for (const t of tags) {
2050
+ if (t.startsWith(tagKeyPrefix)) {
2051
+ return t.slice(tagKeyPrefix.length);
2052
+ }
2053
+ }
2054
+ return null;
2055
+ }
2056
+ }