agent-working-memory 0.11.0 → 0.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -297
- package/dist/adapters/claude-code.d.ts.map +1 -1
- package/dist/adapters/claude-code.js +63 -3
- package/dist/adapters/claude-code.js.map +1 -1
- package/dist/adapters/common.d.ts.map +1 -1
- package/dist/adapters/common.js +358 -306
- package/dist/adapters/common.js.map +1 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +29 -7
- package/dist/api/routes.js.map +1 -1
- package/dist/coordination/routes.d.ts.map +1 -1
- package/dist/coordination/routes.js +174 -170
- package/dist/coordination/routes.js.map +1 -1
- package/dist/core/embeddings.d.ts.map +1 -1
- package/dist/core/embeddings.js +4 -1
- package/dist/core/embeddings.js.map +1 -1
- package/dist/core/entity-extract.d.ts +3 -0
- package/dist/core/entity-extract.d.ts.map +1 -0
- package/dist/core/entity-extract.js +47 -0
- package/dist/core/entity-extract.js.map +1 -0
- package/dist/core/format-recall.d.ts +16 -0
- package/dist/core/format-recall.d.ts.map +1 -0
- package/dist/core/format-recall.js +24 -0
- package/dist/core/format-recall.js.map +1 -0
- package/dist/core/query-expander.js +1 -1
- package/dist/core/query-expander.js.map +1 -1
- package/dist/core/reranker.js +1 -1
- package/dist/core/reranker.js.map +1 -1
- package/dist/core/salience.d.ts.map +1 -1
- package/dist/core/salience.js +14 -2
- package/dist/core/salience.js.map +1 -1
- package/dist/core/whoami.d.ts +24 -0
- package/dist/core/whoami.d.ts.map +1 -0
- package/dist/core/whoami.js +66 -0
- package/dist/core/whoami.js.map +1 -0
- package/dist/core/write-pipeline.d.ts +9 -0
- package/dist/core/write-pipeline.d.ts.map +1 -1
- package/dist/core/write-pipeline.js +109 -68
- package/dist/core/write-pipeline.js.map +1 -1
- package/dist/core/write-telemetry.d.ts +33 -0
- package/dist/core/write-telemetry.d.ts.map +1 -0
- package/dist/core/write-telemetry.js +110 -0
- package/dist/core/write-telemetry.js.map +1 -0
- package/dist/engine/activation.d.ts +22 -12
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +133 -17
- package/dist/engine/activation.js.map +1 -1
- package/dist/engine/consolidation-scheduler.d.ts +1 -1
- package/dist/engine/consolidation-scheduler.js +1 -1
- package/dist/engine/consolidation.d.ts +1 -0
- package/dist/engine/consolidation.d.ts.map +1 -1
- package/dist/engine/consolidation.js +18 -0
- package/dist/engine/consolidation.js.map +1 -1
- package/dist/engine/eval.d.ts.map +1 -1
- package/dist/engine/eval.js +5 -1
- package/dist/engine/eval.js.map +1 -1
- package/dist/hooks/sidecar.d.ts +26 -0
- package/dist/hooks/sidecar.d.ts.map +1 -1
- package/dist/hooks/sidecar.js +30 -0
- package/dist/hooks/sidecar.js.map +1 -1
- package/dist/index.js +20 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +2 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +222 -108
- package/dist/mcp.js.map +1 -1
- package/dist/recipes/index.d.ts +57 -0
- package/dist/recipes/index.d.ts.map +1 -0
- package/dist/recipes/index.js +81 -0
- package/dist/recipes/index.js.map +1 -0
- package/dist/storage/pglite-schema.d.ts.map +1 -1
- package/dist/storage/pglite-schema.js +27 -0
- package/dist/storage/pglite-schema.js.map +1 -1
- package/dist/storage/pglite.d.ts +5 -0
- package/dist/storage/pglite.d.ts.map +1 -1
- package/dist/storage/pglite.js +180 -138
- package/dist/storage/pglite.js.map +1 -1
- package/dist/storage/postgres.d.ts +5 -0
- package/dist/storage/postgres.d.ts.map +1 -1
- package/dist/storage/postgres.js +180 -138
- package/dist/storage/postgres.js.map +1 -1
- package/dist/storage/sqlite.d.ts +9 -0
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +394 -326
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types/engram.d.ts +14 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/dist/types/engram.js.map +1 -1
- package/package.json +1 -1
- package/src/adapters/claude-code.ts +66 -3
- package/src/adapters/common.ts +567 -515
- package/src/api/routes.ts +999 -971
- package/src/coordination/routes.ts +2155 -2150
- package/src/core/embeddings.ts +4 -1
- package/src/core/entity-extract.ts +47 -0
- package/src/core/format-recall.ts +25 -0
- package/src/core/query-expander.ts +1 -1
- package/src/core/reranker.ts +1 -1
- package/src/core/salience.ts +529 -514
- package/src/core/whoami.ts +92 -0
- package/src/core/write-pipeline.ts +60 -8
- package/src/core/write-telemetry.ts +131 -0
- package/src/engine/activation.ts +1468 -1369
- package/src/engine/consolidation-scheduler.ts +1 -1
- package/src/engine/consolidation.ts +887 -869
- package/src/engine/eval.ts +6 -1
- package/src/hooks/sidecar.ts +55 -0
- package/src/index.ts +248 -227
- package/src/mcp.ts +1387 -1270
- package/src/recipes/index.ts +125 -0
- package/src/storage/pglite-schema.ts +27 -0
- package/src/storage/pglite.ts +1420 -1372
- package/src/storage/postgres.ts +1523 -1475
- package/src/storage/sqlite.ts +1936 -1861
- package/src/types/engram.ts +22 -0
package/src/storage/sqlite.ts
CHANGED
|
@@ -1,1861 +1,1936 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* SQLite storage layer — persistence for engrams, associations, and eval events.
|
|
5
|
-
*
|
|
6
|
-
* Uses better-sqlite3 for synchronous, fast, embedded storage.
|
|
7
|
-
* FTS5 provides BM25 full-text search for the activation pipeline.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import Database from 'better-sqlite3';
|
|
11
|
-
import { randomUUID } from 'node:crypto';
|
|
12
|
-
import { cosineSimilarity } from '../core/embeddings.js';
|
|
13
|
-
import type {
|
|
14
|
-
Engram, EngramCreate, EngramStage, Association, AssociationType,
|
|
15
|
-
SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
|
|
16
|
-
RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
|
|
17
|
-
ConsciousState, AutoCheckpoint, CheckpointRow,
|
|
18
|
-
} from '../types/index.js';
|
|
19
|
-
|
|
20
|
-
/** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
|
|
21
|
-
function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
|
|
22
|
-
if (buf instanceof ArrayBuffer) return new Float32Array(buf);
|
|
23
|
-
// Node Buffer may share an underlying ArrayBuffer — slice to the exact region
|
|
24
|
-
const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
25
|
-
return new Float32Array(ab);
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
|
|
29
|
-
surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* In-memory slim entry — the minimum data the activation pipeline's pre-filter
|
|
34
|
-
* pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
|
|
35
|
-
* Float32Array conversion on every recall. ~22 bytes overhead per entry plus
|
|
36
|
-
* the embedding (~1.5KB), so ~15MB at 10K engrams.
|
|
37
|
-
*/
|
|
38
|
-
type SlimCacheEntry = {
|
|
39
|
-
id: string;
|
|
40
|
-
agentId: string;
|
|
41
|
-
concept: string;
|
|
42
|
-
embedding: number[] | null;
|
|
43
|
-
stage: EngramStage;
|
|
44
|
-
retracted: boolean;
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
export class EngramStore {
|
|
48
|
-
private db: Database.Database;
|
|
49
|
-
private walTimer: ReturnType<typeof setInterval> | null = null;
|
|
50
|
-
|
|
51
|
-
// Slim cache for the activation pipeline pre-filter. Populated lazily on
|
|
52
|
-
// first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
|
|
53
|
-
// sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
|
|
54
|
-
// (for A/B testing or if a regression appears).
|
|
55
|
-
private slimCache: Map<string, SlimCacheEntry> = new Map();
|
|
56
|
-
private slimCachePopulated: boolean = false;
|
|
57
|
-
private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
|
|
58
|
-
|
|
59
|
-
constructor(dbPath: string = 'memory.db') {
|
|
60
|
-
this.db = new Database(dbPath);
|
|
61
|
-
this.db.pragma('journal_mode = WAL');
|
|
62
|
-
this.db.pragma('foreign_keys = ON');
|
|
63
|
-
this.db.pragma('busy_timeout = 5000');
|
|
64
|
-
this.db.pragma('synchronous = NORMAL');
|
|
65
|
-
this.db.pragma('wal_autocheckpoint = 1000');
|
|
66
|
-
this.init();
|
|
67
|
-
this.startWalCheckpointTimer();
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// --- Slim cache management ---
|
|
71
|
-
|
|
72
|
-
/** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
|
|
73
|
-
private ensureSlimCachePopulated(): void {
|
|
74
|
-
if (this.slimCachePopulated || !this.slimCacheEnabled) return;
|
|
75
|
-
const rows = this.db.prepare(
|
|
76
|
-
'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
|
|
77
|
-
).all() as any[];
|
|
78
|
-
for (const r of rows) {
|
|
79
|
-
this.slimCache.set(r.id as string, {
|
|
80
|
-
id: r.id as string,
|
|
81
|
-
agentId: r.agent_id as string,
|
|
82
|
-
concept: r.concept as string,
|
|
83
|
-
embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
|
|
84
|
-
stage: r.stage as EngramStage,
|
|
85
|
-
retracted: !!r.retracted,
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
this.slimCachePopulated = true;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** Add a new engram to the slim cache. Called from createEngram. */
|
|
92
|
-
private cacheAdd(entry: SlimCacheEntry): void {
|
|
93
|
-
if (!this.slimCacheEnabled) return;
|
|
94
|
-
this.slimCache.set(entry.id, entry);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
private cacheUpdateStage(id: string, stage: EngramStage): void {
|
|
98
|
-
if (!this.slimCacheEnabled) return;
|
|
99
|
-
const e = this.slimCache.get(id);
|
|
100
|
-
if (e) e.stage = stage;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
private cacheUpdateEmbedding(id: string, embedding: number[]): void {
|
|
104
|
-
if (!this.slimCacheEnabled) return;
|
|
105
|
-
const e = this.slimCache.get(id);
|
|
106
|
-
if (e) e.embedding = embedding;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
private cacheRetract(id: string): void {
|
|
110
|
-
if (!this.slimCacheEnabled) return;
|
|
111
|
-
const e = this.slimCache.get(id);
|
|
112
|
-
if (e) e.retracted = true;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
private cacheRemove(id: string): void {
|
|
116
|
-
if (!this.slimCacheEnabled) return;
|
|
117
|
-
this.slimCache.delete(id);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/** Reset cache (used by tests + after timeWarp/bulk operations). */
|
|
121
|
-
resetSlimCache(): void {
|
|
122
|
-
this.slimCache.clear();
|
|
123
|
-
this.slimCachePopulated = false;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Eager slim-cache populate — public entry point so process startup can warm
|
|
128
|
-
* the cache before the first user recall. Otherwise the first recall pays
|
|
129
|
-
* a ~600ms one-time SQL fetch + embedding deserialization.
|
|
130
|
-
*/
|
|
131
|
-
warmSlimCache(): void {
|
|
132
|
-
this.ensureSlimCachePopulated();
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Inspect cache state — used for diagnostics + tests. */
|
|
136
|
-
getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
|
|
137
|
-
return {
|
|
138
|
-
populated: this.slimCachePopulated,
|
|
139
|
-
size: this.slimCache.size,
|
|
140
|
-
enabled: this.slimCacheEnabled,
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/** Expose the raw database handle for the coordination module. */
|
|
145
|
-
getDb(): Database.Database {
|
|
146
|
-
return this.db;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
/** Run PRAGMA quick_check and return true if DB is healthy. */
|
|
150
|
-
integrityCheck(): { ok: boolean; result: string } {
|
|
151
|
-
try {
|
|
152
|
-
const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
|
|
153
|
-
const result = rows[0]?.quick_check ?? 'unknown';
|
|
154
|
-
return { ok: result === 'ok', result };
|
|
155
|
-
} catch (err) {
|
|
156
|
-
return { ok: false, result: (err as Error).message };
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/** Hot backup using SQLite backup API. Returns the backup path. */
|
|
161
|
-
backup(destPath: string): void {
|
|
162
|
-
this.db.backup(destPath);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/** Flush WAL to main database file. */
|
|
166
|
-
walCheckpoint(): void {
|
|
167
|
-
try {
|
|
168
|
-
this.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
169
|
-
} catch {
|
|
170
|
-
// Checkpoint can fail if another connection holds the DB; non-fatal
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
/** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
|
|
175
|
-
private startWalCheckpointTimer(): void {
|
|
176
|
-
this.walTimer = setInterval(() => {
|
|
177
|
-
this.walCheckpoint();
|
|
178
|
-
}, 5 * 60 * 1000);
|
|
179
|
-
this.walTimer.unref();
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/** Stop the WAL checkpoint timer (call before close). */
|
|
183
|
-
stopWalCheckpointTimer(): void {
|
|
184
|
-
if (this.walTimer) {
|
|
185
|
-
clearInterval(this.walTimer);
|
|
186
|
-
this.walTimer = null;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
private init(): void {
|
|
191
|
-
this.db.exec(`
|
|
192
|
-
CREATE TABLE IF NOT EXISTS engrams (
|
|
193
|
-
id TEXT PRIMARY KEY,
|
|
194
|
-
agent_id TEXT NOT NULL,
|
|
195
|
-
concept TEXT NOT NULL,
|
|
196
|
-
content TEXT NOT NULL,
|
|
197
|
-
embedding BLOB,
|
|
198
|
-
confidence REAL NOT NULL DEFAULT 0.5,
|
|
199
|
-
salience REAL NOT NULL DEFAULT 0.5,
|
|
200
|
-
access_count INTEGER NOT NULL DEFAULT 0,
|
|
201
|
-
last_accessed TEXT NOT NULL,
|
|
202
|
-
created_at TEXT NOT NULL,
|
|
203
|
-
salience_features TEXT NOT NULL DEFAULT '{}',
|
|
204
|
-
reason_codes TEXT NOT NULL DEFAULT '[]',
|
|
205
|
-
stage TEXT NOT NULL DEFAULT 'active',
|
|
206
|
-
ttl INTEGER,
|
|
207
|
-
retracted INTEGER NOT NULL DEFAULT 0,
|
|
208
|
-
retracted_by TEXT,
|
|
209
|
-
retracted_at TEXT,
|
|
210
|
-
tags TEXT NOT NULL DEFAULT '[]',
|
|
211
|
-
memory_type TEXT NOT NULL DEFAULT 'unclassified'
|
|
212
|
-
);
|
|
213
|
-
|
|
214
|
-
CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
|
|
215
|
-
CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
|
|
216
|
-
CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
|
|
217
|
-
CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
|
|
218
|
-
|
|
219
|
-
CREATE TABLE IF NOT EXISTS associations (
|
|
220
|
-
id TEXT PRIMARY KEY,
|
|
221
|
-
from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
|
|
222
|
-
to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
|
|
223
|
-
weight REAL NOT NULL DEFAULT 0.1,
|
|
224
|
-
confidence REAL NOT NULL DEFAULT 0.5,
|
|
225
|
-
type TEXT NOT NULL DEFAULT 'hebbian',
|
|
226
|
-
activation_count INTEGER NOT NULL DEFAULT 0,
|
|
227
|
-
created_at TEXT NOT NULL,
|
|
228
|
-
last_activated TEXT NOT NULL
|
|
229
|
-
);
|
|
230
|
-
|
|
231
|
-
CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
|
|
232
|
-
CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
|
|
233
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
|
|
234
|
-
|
|
235
|
-
CREATE TABLE IF NOT EXISTS agents (
|
|
236
|
-
id TEXT PRIMARY KEY,
|
|
237
|
-
name TEXT NOT NULL,
|
|
238
|
-
created_at TEXT NOT NULL,
|
|
239
|
-
config TEXT NOT NULL DEFAULT '{}'
|
|
240
|
-
);
|
|
241
|
-
|
|
242
|
-
-- FTS5 for full-text search (BM25 ranking built in)
|
|
243
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
|
|
244
|
-
concept, content, tags,
|
|
245
|
-
content=engrams,
|
|
246
|
-
content_rowid=rowid
|
|
247
|
-
);
|
|
248
|
-
|
|
249
|
-
-- Triggers to keep FTS in sync
|
|
250
|
-
CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
|
|
251
|
-
INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
|
|
252
|
-
END;
|
|
253
|
-
CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
|
|
254
|
-
INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
|
|
255
|
-
END;
|
|
256
|
-
CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
|
|
257
|
-
INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
|
|
258
|
-
INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
|
|
259
|
-
END;
|
|
260
|
-
|
|
261
|
-
-- Eval event logs
|
|
262
|
-
CREATE TABLE IF NOT EXISTS activation_events (
|
|
263
|
-
id TEXT PRIMARY KEY,
|
|
264
|
-
agent_id TEXT NOT NULL,
|
|
265
|
-
timestamp TEXT NOT NULL,
|
|
266
|
-
context TEXT NOT NULL,
|
|
267
|
-
results_returned INTEGER NOT NULL,
|
|
268
|
-
top_score REAL,
|
|
269
|
-
latency_ms REAL NOT NULL,
|
|
270
|
-
engram_ids TEXT NOT NULL DEFAULT '[]'
|
|
271
|
-
);
|
|
272
|
-
|
|
273
|
-
CREATE TABLE IF NOT EXISTS staging_events (
|
|
274
|
-
engram_id TEXT NOT NULL,
|
|
275
|
-
agent_id TEXT NOT NULL,
|
|
276
|
-
action TEXT NOT NULL,
|
|
277
|
-
resonance_score REAL,
|
|
278
|
-
timestamp TEXT NOT NULL,
|
|
279
|
-
age_ms INTEGER NOT NULL
|
|
280
|
-
);
|
|
281
|
-
|
|
282
|
-
CREATE TABLE IF NOT EXISTS retrieval_feedback (
|
|
283
|
-
id TEXT PRIMARY KEY,
|
|
284
|
-
activation_event_id TEXT,
|
|
285
|
-
engram_id TEXT NOT NULL,
|
|
286
|
-
useful INTEGER NOT NULL,
|
|
287
|
-
context TEXT,
|
|
288
|
-
timestamp TEXT NOT NULL
|
|
289
|
-
);
|
|
290
|
-
|
|
291
|
-
CREATE TABLE IF NOT EXISTS episodes (
|
|
292
|
-
id TEXT PRIMARY KEY,
|
|
293
|
-
agent_id TEXT NOT NULL,
|
|
294
|
-
label TEXT NOT NULL,
|
|
295
|
-
embedding BLOB,
|
|
296
|
-
engram_count INTEGER NOT NULL DEFAULT 0,
|
|
297
|
-
start_time TEXT NOT NULL,
|
|
298
|
-
end_time TEXT NOT NULL,
|
|
299
|
-
created_at TEXT NOT NULL
|
|
300
|
-
);
|
|
301
|
-
|
|
302
|
-
CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
|
|
303
|
-
CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
|
|
304
|
-
`);
|
|
305
|
-
|
|
306
|
-
// Migration: add episode_id column if missing
|
|
307
|
-
try {
|
|
308
|
-
this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
|
|
309
|
-
} catch {
|
|
310
|
-
this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
// Migration: add task management columns if missing
|
|
314
|
-
try {
|
|
315
|
-
this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
|
|
316
|
-
} catch {
|
|
317
|
-
this.db.exec(`
|
|
318
|
-
ALTER TABLE engrams ADD COLUMN task_status TEXT;
|
|
319
|
-
ALTER TABLE engrams ADD COLUMN task_priority TEXT;
|
|
320
|
-
ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
|
|
321
|
-
`);
|
|
322
|
-
this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
// Migration: add memory_class and supersession columns if missing
|
|
326
|
-
try {
|
|
327
|
-
this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
|
|
328
|
-
} catch {
|
|
329
|
-
this.db.exec(`
|
|
330
|
-
ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
|
|
331
|
-
ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
|
|
332
|
-
ALTER TABLE engrams ADD COLUMN supersedes TEXT;
|
|
333
|
-
`);
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
// Migration: add embedding_model for version tracking (prevents drift on model change)
|
|
337
|
-
try {
|
|
338
|
-
this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
|
|
339
|
-
} catch {
|
|
340
|
-
this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
//
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
this.
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
/**
|
|
751
|
-
*
|
|
752
|
-
*
|
|
753
|
-
*
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
//
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
*
|
|
792
|
-
*
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
sql +=
|
|
887
|
-
|
|
888
|
-
}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
*
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
const
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
).get(
|
|
1071
|
-
return row
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
).all(...chunk, ...chunk) as
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
return
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
return
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
const row = this.db.prepare(
|
|
1223
|
-
|
|
1224
|
-
).get(
|
|
1225
|
-
return row.
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
);
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
`).
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
}
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
return {
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
: null
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
return
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
}
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
return
|
|
1439
|
-
}
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
const
|
|
1443
|
-
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
const
|
|
1448
|
-
'SELECT
|
|
1449
|
-
).
|
|
1450
|
-
return
|
|
1451
|
-
}
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
const
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
const
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
last_activity_at = excluded.last_activity_at,
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
this.
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
*
|
|
1700
|
-
*
|
|
1701
|
-
*
|
|
1702
|
-
*
|
|
1703
|
-
*
|
|
1704
|
-
*/
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
const
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
*
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
//
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
const
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
)
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
}
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* SQLite storage layer — persistence for engrams, associations, and eval events.
|
|
5
|
+
*
|
|
6
|
+
* Uses better-sqlite3 for synchronous, fast, embedded storage.
|
|
7
|
+
* FTS5 provides BM25 full-text search for the activation pipeline.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import Database from 'better-sqlite3';
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
12
|
+
import { cosineSimilarity } from '../core/embeddings.js';
|
|
13
|
+
import type {
|
|
14
|
+
Engram, EngramCreate, EngramStage, Association, AssociationType,
|
|
15
|
+
SearchQuery, SalienceFeatures, ActivationEvent, StagingEvent,
|
|
16
|
+
RetrievalFeedbackEvent, Episode, TaskStatus, TaskPriority, MemoryClass, MemoryType,
|
|
17
|
+
ConsciousState, AutoCheckpoint, CheckpointRow,
|
|
18
|
+
} from '../types/index.js';
|
|
19
|
+
|
|
20
|
+
/** Safely convert a Node Buffer to Float32Array, respecting byteOffset/byteLength. */
|
|
21
|
+
function bufferToFloat32Array(buf: Buffer | ArrayBuffer): Float32Array {
|
|
22
|
+
if (buf instanceof ArrayBuffer) return new Float32Array(buf);
|
|
23
|
+
// Node Buffer may share an underlying ArrayBuffer — slice to the exact region
|
|
24
|
+
const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
25
|
+
return new Float32Array(ab);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
|
|
29
|
+
surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* In-memory slim entry — the minimum data the activation pipeline's pre-filter
|
|
34
|
+
* pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
|
|
35
|
+
* Float32Array conversion on every recall. ~22 bytes overhead per entry plus
|
|
36
|
+
* the embedding (~1.5KB), so ~15MB at 10K engrams.
|
|
37
|
+
*/
|
|
38
|
+
type SlimCacheEntry = {
|
|
39
|
+
id: string;
|
|
40
|
+
agentId: string;
|
|
41
|
+
concept: string;
|
|
42
|
+
embedding: number[] | null;
|
|
43
|
+
stage: EngramStage;
|
|
44
|
+
retracted: boolean;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export class EngramStore {
|
|
48
|
+
private db: Database.Database;
|
|
49
|
+
private walTimer: ReturnType<typeof setInterval> | null = null;
|
|
50
|
+
|
|
51
|
+
// Slim cache for the activation pipeline pre-filter. Populated lazily on
|
|
52
|
+
// first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
|
|
53
|
+
// sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
|
|
54
|
+
// (for A/B testing or if a regression appears).
|
|
55
|
+
private slimCache: Map<string, SlimCacheEntry> = new Map();
|
|
56
|
+
private slimCachePopulated: boolean = false;
|
|
57
|
+
private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
|
|
58
|
+
|
|
59
|
+
constructor(dbPath: string = 'memory.db') {
|
|
60
|
+
this.db = new Database(dbPath);
|
|
61
|
+
this.db.pragma('journal_mode = WAL');
|
|
62
|
+
this.db.pragma('foreign_keys = ON');
|
|
63
|
+
this.db.pragma('busy_timeout = 5000');
|
|
64
|
+
this.db.pragma('synchronous = NORMAL');
|
|
65
|
+
this.db.pragma('wal_autocheckpoint = 1000');
|
|
66
|
+
this.init();
|
|
67
|
+
this.startWalCheckpointTimer();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// --- Slim cache management ---
|
|
71
|
+
|
|
72
|
+
/** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
|
|
73
|
+
private ensureSlimCachePopulated(): void {
|
|
74
|
+
if (this.slimCachePopulated || !this.slimCacheEnabled) return;
|
|
75
|
+
const rows = this.db.prepare(
|
|
76
|
+
'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
|
|
77
|
+
).all() as any[];
|
|
78
|
+
for (const r of rows) {
|
|
79
|
+
this.slimCache.set(r.id as string, {
|
|
80
|
+
id: r.id as string,
|
|
81
|
+
agentId: r.agent_id as string,
|
|
82
|
+
concept: r.concept as string,
|
|
83
|
+
embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
|
|
84
|
+
stage: r.stage as EngramStage,
|
|
85
|
+
retracted: !!r.retracted,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
this.slimCachePopulated = true;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Add a new engram to the slim cache. Called from createEngram. */
|
|
92
|
+
private cacheAdd(entry: SlimCacheEntry): void {
|
|
93
|
+
if (!this.slimCacheEnabled) return;
|
|
94
|
+
this.slimCache.set(entry.id, entry);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private cacheUpdateStage(id: string, stage: EngramStage): void {
|
|
98
|
+
if (!this.slimCacheEnabled) return;
|
|
99
|
+
const e = this.slimCache.get(id);
|
|
100
|
+
if (e) e.stage = stage;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private cacheUpdateEmbedding(id: string, embedding: number[]): void {
|
|
104
|
+
if (!this.slimCacheEnabled) return;
|
|
105
|
+
const e = this.slimCache.get(id);
|
|
106
|
+
if (e) e.embedding = embedding;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private cacheRetract(id: string): void {
|
|
110
|
+
if (!this.slimCacheEnabled) return;
|
|
111
|
+
const e = this.slimCache.get(id);
|
|
112
|
+
if (e) e.retracted = true;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
private cacheRemove(id: string): void {
|
|
116
|
+
if (!this.slimCacheEnabled) return;
|
|
117
|
+
this.slimCache.delete(id);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Reset cache (used by tests + after timeWarp/bulk operations). */
|
|
121
|
+
resetSlimCache(): void {
|
|
122
|
+
this.slimCache.clear();
|
|
123
|
+
this.slimCachePopulated = false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Eager slim-cache populate — public entry point so process startup can warm
|
|
128
|
+
* the cache before the first user recall. Otherwise the first recall pays
|
|
129
|
+
* a ~600ms one-time SQL fetch + embedding deserialization.
|
|
130
|
+
*/
|
|
131
|
+
warmSlimCache(): void {
|
|
132
|
+
this.ensureSlimCachePopulated();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Inspect cache state — used for diagnostics + tests. */
|
|
136
|
+
getSlimCacheStats(): { populated: boolean; size: number; enabled: boolean } {
|
|
137
|
+
return {
|
|
138
|
+
populated: this.slimCachePopulated,
|
|
139
|
+
size: this.slimCache.size,
|
|
140
|
+
enabled: this.slimCacheEnabled,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Expose the raw database handle for the coordination module. */
|
|
145
|
+
getDb(): Database.Database {
|
|
146
|
+
return this.db;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Run PRAGMA quick_check and return true if DB is healthy. */
|
|
150
|
+
integrityCheck(): { ok: boolean; result: string } {
|
|
151
|
+
try {
|
|
152
|
+
const rows = this.db.pragma('quick_check') as Array<{ quick_check: string }>;
|
|
153
|
+
const result = rows[0]?.quick_check ?? 'unknown';
|
|
154
|
+
return { ok: result === 'ok', result };
|
|
155
|
+
} catch (err) {
|
|
156
|
+
return { ok: false, result: (err as Error).message };
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Hot backup using SQLite backup API. Returns the backup path. */
|
|
161
|
+
backup(destPath: string): void {
|
|
162
|
+
this.db.backup(destPath);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Flush WAL to main database file. */
|
|
166
|
+
walCheckpoint(): void {
|
|
167
|
+
try {
|
|
168
|
+
this.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
169
|
+
} catch {
|
|
170
|
+
// Checkpoint can fail if another connection holds the DB; non-fatal
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Start periodic WAL checkpoint every 5 minutes to prevent unbounded WAL growth. */
|
|
175
|
+
private startWalCheckpointTimer(): void {
|
|
176
|
+
this.walTimer = setInterval(() => {
|
|
177
|
+
this.walCheckpoint();
|
|
178
|
+
}, 5 * 60 * 1000);
|
|
179
|
+
this.walTimer.unref();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Stop the WAL checkpoint timer (call before close). */
|
|
183
|
+
stopWalCheckpointTimer(): void {
|
|
184
|
+
if (this.walTimer) {
|
|
185
|
+
clearInterval(this.walTimer);
|
|
186
|
+
this.walTimer = null;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private init(): void {
|
|
191
|
+
this.db.exec(`
|
|
192
|
+
CREATE TABLE IF NOT EXISTS engrams (
|
|
193
|
+
id TEXT PRIMARY KEY,
|
|
194
|
+
agent_id TEXT NOT NULL,
|
|
195
|
+
concept TEXT NOT NULL,
|
|
196
|
+
content TEXT NOT NULL,
|
|
197
|
+
embedding BLOB,
|
|
198
|
+
confidence REAL NOT NULL DEFAULT 0.5,
|
|
199
|
+
salience REAL NOT NULL DEFAULT 0.5,
|
|
200
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
201
|
+
last_accessed TEXT NOT NULL,
|
|
202
|
+
created_at TEXT NOT NULL,
|
|
203
|
+
salience_features TEXT NOT NULL DEFAULT '{}',
|
|
204
|
+
reason_codes TEXT NOT NULL DEFAULT '[]',
|
|
205
|
+
stage TEXT NOT NULL DEFAULT 'active',
|
|
206
|
+
ttl INTEGER,
|
|
207
|
+
retracted INTEGER NOT NULL DEFAULT 0,
|
|
208
|
+
retracted_by TEXT,
|
|
209
|
+
retracted_at TEXT,
|
|
210
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
211
|
+
memory_type TEXT NOT NULL DEFAULT 'unclassified'
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
CREATE INDEX IF NOT EXISTS idx_engrams_agent ON engrams(agent_id);
|
|
215
|
+
CREATE INDEX IF NOT EXISTS idx_engrams_stage ON engrams(agent_id, stage);
|
|
216
|
+
CREATE INDEX IF NOT EXISTS idx_engrams_concept ON engrams(concept);
|
|
217
|
+
CREATE INDEX IF NOT EXISTS idx_engrams_retracted ON engrams(agent_id, retracted);
|
|
218
|
+
|
|
219
|
+
CREATE TABLE IF NOT EXISTS associations (
|
|
220
|
+
id TEXT PRIMARY KEY,
|
|
221
|
+
from_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
|
|
222
|
+
to_engram_id TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
|
|
223
|
+
weight REAL NOT NULL DEFAULT 0.1,
|
|
224
|
+
confidence REAL NOT NULL DEFAULT 0.5,
|
|
225
|
+
type TEXT NOT NULL DEFAULT 'hebbian',
|
|
226
|
+
activation_count INTEGER NOT NULL DEFAULT 0,
|
|
227
|
+
created_at TEXT NOT NULL,
|
|
228
|
+
last_activated TEXT NOT NULL
|
|
229
|
+
);
|
|
230
|
+
|
|
231
|
+
CREATE INDEX IF NOT EXISTS idx_assoc_from ON associations(from_engram_id);
|
|
232
|
+
CREATE INDEX IF NOT EXISTS idx_assoc_to ON associations(to_engram_id);
|
|
233
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
|
|
234
|
+
|
|
235
|
+
CREATE TABLE IF NOT EXISTS agents (
|
|
236
|
+
id TEXT PRIMARY KEY,
|
|
237
|
+
name TEXT NOT NULL,
|
|
238
|
+
created_at TEXT NOT NULL,
|
|
239
|
+
config TEXT NOT NULL DEFAULT '{}'
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
-- FTS5 for full-text search (BM25 ranking built in)
|
|
243
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS engrams_fts USING fts5(
|
|
244
|
+
concept, content, tags,
|
|
245
|
+
content=engrams,
|
|
246
|
+
content_rowid=rowid
|
|
247
|
+
);
|
|
248
|
+
|
|
249
|
+
-- Triggers to keep FTS in sync
|
|
250
|
+
CREATE TRIGGER IF NOT EXISTS engrams_ai AFTER INSERT ON engrams BEGIN
|
|
251
|
+
INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
|
|
252
|
+
END;
|
|
253
|
+
CREATE TRIGGER IF NOT EXISTS engrams_ad AFTER DELETE ON engrams BEGIN
|
|
254
|
+
INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
|
|
255
|
+
END;
|
|
256
|
+
CREATE TRIGGER IF NOT EXISTS engrams_au AFTER UPDATE ON engrams BEGIN
|
|
257
|
+
INSERT INTO engrams_fts(engrams_fts, rowid, concept, content, tags) VALUES('delete', old.rowid, old.concept, old.content, old.tags);
|
|
258
|
+
INSERT INTO engrams_fts(rowid, concept, content, tags) VALUES (new.rowid, new.concept, new.content, new.tags);
|
|
259
|
+
END;
|
|
260
|
+
|
|
261
|
+
-- Eval event logs
|
|
262
|
+
CREATE TABLE IF NOT EXISTS activation_events (
|
|
263
|
+
id TEXT PRIMARY KEY,
|
|
264
|
+
agent_id TEXT NOT NULL,
|
|
265
|
+
timestamp TEXT NOT NULL,
|
|
266
|
+
context TEXT NOT NULL,
|
|
267
|
+
results_returned INTEGER NOT NULL,
|
|
268
|
+
top_score REAL,
|
|
269
|
+
latency_ms REAL NOT NULL,
|
|
270
|
+
engram_ids TEXT NOT NULL DEFAULT '[]'
|
|
271
|
+
);
|
|
272
|
+
|
|
273
|
+
CREATE TABLE IF NOT EXISTS staging_events (
|
|
274
|
+
engram_id TEXT NOT NULL,
|
|
275
|
+
agent_id TEXT NOT NULL,
|
|
276
|
+
action TEXT NOT NULL,
|
|
277
|
+
resonance_score REAL,
|
|
278
|
+
timestamp TEXT NOT NULL,
|
|
279
|
+
age_ms INTEGER NOT NULL
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
CREATE TABLE IF NOT EXISTS retrieval_feedback (
|
|
283
|
+
id TEXT PRIMARY KEY,
|
|
284
|
+
activation_event_id TEXT,
|
|
285
|
+
engram_id TEXT NOT NULL,
|
|
286
|
+
useful INTEGER NOT NULL,
|
|
287
|
+
context TEXT,
|
|
288
|
+
timestamp TEXT NOT NULL
|
|
289
|
+
);
|
|
290
|
+
|
|
291
|
+
CREATE TABLE IF NOT EXISTS episodes (
|
|
292
|
+
id TEXT PRIMARY KEY,
|
|
293
|
+
agent_id TEXT NOT NULL,
|
|
294
|
+
label TEXT NOT NULL,
|
|
295
|
+
embedding BLOB,
|
|
296
|
+
engram_count INTEGER NOT NULL DEFAULT 0,
|
|
297
|
+
start_time TEXT NOT NULL,
|
|
298
|
+
end_time TEXT NOT NULL,
|
|
299
|
+
created_at TEXT NOT NULL
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
|
|
303
|
+
CREATE INDEX IF NOT EXISTS idx_episodes_time ON episodes(agent_id, end_time);
|
|
304
|
+
`);
|
|
305
|
+
|
|
306
|
+
// Migration: add episode_id column if missing
|
|
307
|
+
try {
|
|
308
|
+
this.db.prepare('SELECT episode_id FROM engrams LIMIT 0').get();
|
|
309
|
+
} catch {
|
|
310
|
+
this.db.exec('ALTER TABLE engrams ADD COLUMN episode_id TEXT');
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Migration: add task management columns if missing
|
|
314
|
+
try {
|
|
315
|
+
this.db.prepare('SELECT task_status FROM engrams LIMIT 0').get();
|
|
316
|
+
} catch {
|
|
317
|
+
this.db.exec(`
|
|
318
|
+
ALTER TABLE engrams ADD COLUMN task_status TEXT;
|
|
319
|
+
ALTER TABLE engrams ADD COLUMN task_priority TEXT;
|
|
320
|
+
ALTER TABLE engrams ADD COLUMN blocked_by TEXT;
|
|
321
|
+
`);
|
|
322
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_engrams_task ON engrams(agent_id, task_status)');
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// Migration: add memory_class and supersession columns if missing
|
|
326
|
+
try {
|
|
327
|
+
this.db.prepare('SELECT memory_class FROM engrams LIMIT 0').get();
|
|
328
|
+
} catch {
|
|
329
|
+
this.db.exec(`
|
|
330
|
+
ALTER TABLE engrams ADD COLUMN memory_class TEXT NOT NULL DEFAULT 'working';
|
|
331
|
+
ALTER TABLE engrams ADD COLUMN superseded_by TEXT;
|
|
332
|
+
ALTER TABLE engrams ADD COLUMN supersedes TEXT;
|
|
333
|
+
`);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Migration: add embedding_model for version tracking (prevents drift on model change)
|
|
337
|
+
try {
|
|
338
|
+
this.db.prepare('SELECT embedding_model FROM engrams LIMIT 0').get();
|
|
339
|
+
} catch {
|
|
340
|
+
this.db.exec(`ALTER TABLE engrams ADD COLUMN embedding_model TEXT`);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Entity inverted index + alias table (D9, 2026-07-30) — write-time
|
|
344
|
+
// bookkeeping only; retrieval unchanged until D11. Idempotent CREATEs.
|
|
345
|
+
this.db.exec(`
|
|
346
|
+
CREATE TABLE IF NOT EXISTS entity_mentions (
|
|
347
|
+
entity TEXT NOT NULL,
|
|
348
|
+
engram_id TEXT NOT NULL,
|
|
349
|
+
agent_id TEXT NOT NULL,
|
|
350
|
+
PRIMARY KEY (entity, engram_id)
|
|
351
|
+
);
|
|
352
|
+
CREATE INDEX IF NOT EXISTS idx_entity_mentions_agent ON entity_mentions(agent_id, entity);
|
|
353
|
+
CREATE TABLE IF NOT EXISTS entity_aliases (
|
|
354
|
+
alias TEXT PRIMARY KEY,
|
|
355
|
+
entity TEXT NOT NULL
|
|
356
|
+
);
|
|
357
|
+
`);
|
|
358
|
+
|
|
359
|
+
// Migration: memory-spine provenance + temporal validity (D5/D8, 2026-07-30)
|
|
360
|
+
try {
|
|
361
|
+
this.db.prepare('SELECT origin_class FROM engrams LIMIT 0').get();
|
|
362
|
+
} catch {
|
|
363
|
+
this.db.exec(`
|
|
364
|
+
ALTER TABLE engrams ADD COLUMN origin_class TEXT;
|
|
365
|
+
ALTER TABLE engrams ADD COLUMN writer_session TEXT;
|
|
366
|
+
ALTER TABLE engrams ADD COLUMN recipe_id TEXT;
|
|
367
|
+
ALTER TABLE engrams ADD COLUMN valid_from TEXT;
|
|
368
|
+
ALTER TABLE engrams ADD COLUMN valid_to TEXT;
|
|
369
|
+
`);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Migration: add conscious_state table for checkpointing
|
|
373
|
+
this.db.exec(`
|
|
374
|
+
CREATE TABLE IF NOT EXISTS conscious_state (
|
|
375
|
+
agent_id TEXT PRIMARY KEY,
|
|
376
|
+
last_write_id TEXT,
|
|
377
|
+
last_recall_context TEXT,
|
|
378
|
+
last_recall_ids TEXT NOT NULL DEFAULT '[]',
|
|
379
|
+
last_activity_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
380
|
+
write_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
|
|
381
|
+
recall_count_since_consolidation INTEGER NOT NULL DEFAULT 0,
|
|
382
|
+
execution_state TEXT,
|
|
383
|
+
checkpoint_at TEXT,
|
|
384
|
+
last_consolidation_at TEXT,
|
|
385
|
+
last_mini_consolidation_at TEXT,
|
|
386
|
+
consolidation_cycle_count INTEGER NOT NULL DEFAULT 0,
|
|
387
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
388
|
+
)
|
|
389
|
+
`);
|
|
390
|
+
|
|
391
|
+
// Migration: add consolidation_cycle_count if missing (existing DBs)
|
|
392
|
+
try {
|
|
393
|
+
this.db.exec(`ALTER TABLE conscious_state ADD COLUMN consolidation_cycle_count INTEGER NOT NULL DEFAULT 0`);
|
|
394
|
+
} catch { /* column already exists */ }
|
|
395
|
+
|
|
396
|
+
// Migration: add memory_type column if missing
|
|
397
|
+
try {
|
|
398
|
+
this.db.prepare('SELECT memory_type FROM engrams LIMIT 0').get();
|
|
399
|
+
} catch {
|
|
400
|
+
this.db.exec(`ALTER TABLE engrams ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'unclassified'`);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Migration (0.8 Cluster A): sequence column for story-time / chronology.
|
|
404
|
+
// Nullable — existing engrams stay NULL. Partial index keeps the cost low.
|
|
405
|
+
try {
|
|
406
|
+
this.db.prepare('SELECT sequence FROM engrams LIMIT 0').get();
|
|
407
|
+
} catch {
|
|
408
|
+
this.db.exec(`ALTER TABLE engrams ADD COLUMN sequence INTEGER`);
|
|
409
|
+
this.db.exec(
|
|
410
|
+
`CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence
|
|
411
|
+
ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL`
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Migration (0.8 Cluster A): references_json column for typed cross-record
|
|
416
|
+
// links. Wired through HTTP in Cluster D — schema slot only here.
|
|
417
|
+
try {
|
|
418
|
+
this.db.prepare('SELECT references_json FROM engrams LIMIT 0').get();
|
|
419
|
+
} catch {
|
|
420
|
+
this.db.exec(`ALTER TABLE engrams ADD COLUMN references_json TEXT`);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// --- Engram CRUD ---
|
|
425
|
+
|
|
426
|
+
createEngram(input: EngramCreate): Engram {
|
|
427
|
+
const now = new Date().toISOString();
|
|
428
|
+
const id = randomUUID();
|
|
429
|
+
const embeddingBlob = input.embedding
|
|
430
|
+
? Buffer.from(new Float32Array(input.embedding).buffer)
|
|
431
|
+
: null;
|
|
432
|
+
|
|
433
|
+
this.db.prepare(`
|
|
434
|
+
INSERT INTO engrams (id, agent_id, concept, content, embedding, confidence, salience,
|
|
435
|
+
access_count, last_accessed, created_at, salience_features, reason_codes, stage, tags, episode_id,
|
|
436
|
+
ttl, memory_class, supersedes, task_status, task_priority, blocked_by, memory_type,
|
|
437
|
+
sequence, references_json, origin_class, writer_session, recipe_id, valid_from, valid_to)
|
|
438
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
439
|
+
`).run(
|
|
440
|
+
id, input.agentId, input.concept, input.content, embeddingBlob,
|
|
441
|
+
input.confidence ?? 0.5,
|
|
442
|
+
input.salience ?? 0.5,
|
|
443
|
+
now, now,
|
|
444
|
+
JSON.stringify(input.salienceFeatures ?? DEFAULT_SALIENCE_FEATURES),
|
|
445
|
+
JSON.stringify(input.reasonCodes ?? []),
|
|
446
|
+
JSON.stringify(input.tags ?? []),
|
|
447
|
+
input.episodeId ?? null,
|
|
448
|
+
input.ttl ?? null,
|
|
449
|
+
input.memoryClass ?? 'working',
|
|
450
|
+
input.supersedes ?? null,
|
|
451
|
+
input.taskStatus ?? null,
|
|
452
|
+
input.taskPriority ?? null,
|
|
453
|
+
input.blockedBy ?? null,
|
|
454
|
+
input.memoryType ?? 'unclassified',
|
|
455
|
+
input.sequence ?? null,
|
|
456
|
+
input.references && input.references.length > 0
|
|
457
|
+
? JSON.stringify(input.references) : null,
|
|
458
|
+
input.originClass ?? null,
|
|
459
|
+
input.writerSession ?? null,
|
|
460
|
+
input.recipeId ?? null,
|
|
461
|
+
input.validFrom ?? null,
|
|
462
|
+
input.validTo ?? null,
|
|
463
|
+
);
|
|
464
|
+
|
|
465
|
+
// Add to slim cache (skip if not yet populated — first slim fetch will load it)
|
|
466
|
+
if (this.slimCachePopulated) {
|
|
467
|
+
this.cacheAdd({
|
|
468
|
+
id,
|
|
469
|
+
agentId: input.agentId,
|
|
470
|
+
concept: input.concept,
|
|
471
|
+
embedding: input.embedding ?? null,
|
|
472
|
+
stage: 'active',
|
|
473
|
+
retracted: false,
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
return this.getEngram(id)!;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
getEngram(id: string): Engram | null {
|
|
481
|
+
const row = this.db.prepare('SELECT * FROM engrams WHERE id = ?').get(id) as any;
|
|
482
|
+
return row ? this.rowToEngram(row) : null;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
getEngramsByAgent(agentId: string, stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
|
|
486
|
+
let query = 'SELECT * FROM engrams WHERE agent_id = ?';
|
|
487
|
+
const params: any[] = [agentId];
|
|
488
|
+
|
|
489
|
+
if (stage) {
|
|
490
|
+
query += ' AND stage = ?';
|
|
491
|
+
params.push(stage);
|
|
492
|
+
}
|
|
493
|
+
if (!includeRetracted) {
|
|
494
|
+
query += ' AND retracted = 0';
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* Slim variant that returns only (id, concept, embedding) — the minimum needed
|
|
502
|
+
* for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
|
|
503
|
+
* survival check). Avoids materializing the content blob, tag JSON, salience
|
|
504
|
+
* features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
|
|
505
|
+
*
|
|
506
|
+
* Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
|
|
507
|
+
* engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
|
|
508
|
+
* of that is row materialization of fields we don't read in the filter pass.
|
|
509
|
+
*/
|
|
510
|
+
getEngramsByAgentSlim(
|
|
511
|
+
agentId: string,
|
|
512
|
+
stage?: EngramStage,
|
|
513
|
+
includeRetracted: boolean = false
|
|
514
|
+
): Array<{ id: string; concept: string; embedding: number[] | null }> {
|
|
515
|
+
if (this.slimCacheEnabled) {
|
|
516
|
+
this.ensureSlimCachePopulated();
|
|
517
|
+
const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
|
|
518
|
+
for (const entry of this.slimCache.values()) {
|
|
519
|
+
if (entry.agentId !== agentId) continue;
|
|
520
|
+
if (stage && entry.stage !== stage) continue;
|
|
521
|
+
if (!includeRetracted && entry.retracted) continue;
|
|
522
|
+
result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
|
|
523
|
+
}
|
|
524
|
+
return result;
|
|
525
|
+
}
|
|
526
|
+
// Cache disabled — fall back to direct SQL
|
|
527
|
+
let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
|
|
528
|
+
const params: any[] = [agentId];
|
|
529
|
+
|
|
530
|
+
if (stage) {
|
|
531
|
+
query += ' AND stage = ?';
|
|
532
|
+
params.push(stage);
|
|
533
|
+
}
|
|
534
|
+
if (!includeRetracted) {
|
|
535
|
+
query += ' AND retracted = 0';
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
return (this.db.prepare(query).all(...params) as any[]).map(r => ({
|
|
539
|
+
id: r.id as string,
|
|
540
|
+
concept: r.concept as string,
|
|
541
|
+
embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
|
|
542
|
+
}));
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** Slim variant for multi-agent (workspace-scoped) pre-filter. */
|
|
546
|
+
getEngramsByAgentsSlim(
|
|
547
|
+
agentIds: string[],
|
|
548
|
+
stage?: EngramStage,
|
|
549
|
+
includeRetracted: boolean = false
|
|
550
|
+
): Array<{ id: string; concept: string; embedding: number[] | null }> {
|
|
551
|
+
if (agentIds.length === 0) return [];
|
|
552
|
+
if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
|
|
553
|
+
|
|
554
|
+
if (this.slimCacheEnabled) {
|
|
555
|
+
this.ensureSlimCachePopulated();
|
|
556
|
+
const agentSet = new Set(agentIds);
|
|
557
|
+
const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
|
|
558
|
+
for (const entry of this.slimCache.values()) {
|
|
559
|
+
if (!agentSet.has(entry.agentId)) continue;
|
|
560
|
+
if (stage && entry.stage !== stage) continue;
|
|
561
|
+
if (!includeRetracted && entry.retracted) continue;
|
|
562
|
+
result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
|
|
563
|
+
}
|
|
564
|
+
return result;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const placeholders = agentIds.map(() => '?').join(',');
|
|
568
|
+
let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
|
|
569
|
+
const params: any[] = [...agentIds];
|
|
570
|
+
|
|
571
|
+
if (stage) {
|
|
572
|
+
query += ' AND stage = ?';
|
|
573
|
+
params.push(stage);
|
|
574
|
+
}
|
|
575
|
+
if (!includeRetracted) {
|
|
576
|
+
query += ' AND retracted = 0';
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
return (this.db.prepare(query).all(...params) as any[]).map(r => ({
|
|
580
|
+
id: r.id as string,
|
|
581
|
+
concept: r.concept as string,
|
|
582
|
+
embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
|
|
583
|
+
}));
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
|
|
588
|
+
* only the survivors that need deep scoring. Chunks IN-clause queries to stay
|
|
589
|
+
* under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
|
|
590
|
+
*/
|
|
591
|
+
getEngramsByIds(ids: string[]): Engram[] {
|
|
592
|
+
if (ids.length === 0) return [];
|
|
593
|
+
const CHUNK = 800;
|
|
594
|
+
const result: Engram[] = [];
|
|
595
|
+
for (let i = 0; i < ids.length; i += CHUNK) {
|
|
596
|
+
const chunk = ids.slice(i, i + CHUNK);
|
|
597
|
+
const placeholders = chunk.map(() => '?').join(',');
|
|
598
|
+
const rows = this.db.prepare(
|
|
599
|
+
`SELECT * FROM engrams WHERE id IN (${placeholders})`
|
|
600
|
+
).all(...chunk) as any[];
|
|
601
|
+
for (const r of rows) result.push(this.rowToEngram(r));
|
|
602
|
+
}
|
|
603
|
+
return result;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Get engrams across multiple agents (workspace-scoped recall).
|
|
608
|
+
* Used when workspace mode is enabled for hive memory sharing.
|
|
609
|
+
*/
|
|
610
|
+
getEngramsByAgents(agentIds: string[], stage?: EngramStage, includeRetracted: boolean = false): Engram[] {
|
|
611
|
+
if (agentIds.length === 0) return [];
|
|
612
|
+
if (agentIds.length === 1) return this.getEngramsByAgent(agentIds[0], stage, includeRetracted);
|
|
613
|
+
|
|
614
|
+
const placeholders = agentIds.map(() => '?').join(',');
|
|
615
|
+
let query = `SELECT * FROM engrams WHERE agent_id IN (${placeholders})`;
|
|
616
|
+
const params: any[] = [...agentIds];
|
|
617
|
+
|
|
618
|
+
if (stage) {
|
|
619
|
+
query += ' AND stage = ?';
|
|
620
|
+
params.push(stage);
|
|
621
|
+
}
|
|
622
|
+
if (!includeRetracted) {
|
|
623
|
+
query += ' AND retracted = 0';
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* BM25 search across multiple agents (workspace-scoped).
|
|
631
|
+
*/
|
|
632
|
+
searchBM25WithRankMultiAgent(agentIds: string[], query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
|
|
633
|
+
if (agentIds.length === 0) return [];
|
|
634
|
+
if (agentIds.length === 1) return this.searchBM25WithRank(agentIds[0], query, limit);
|
|
635
|
+
|
|
636
|
+
const sanitized = query
|
|
637
|
+
.replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
|
|
638
|
+
.split(/\s+/)
|
|
639
|
+
.filter(w => w.length > 1)
|
|
640
|
+
.map(w => `"${w}"`)
|
|
641
|
+
.join(' OR ');
|
|
642
|
+
|
|
643
|
+
if (!sanitized) return [];
|
|
644
|
+
|
|
645
|
+
// CTE prefilter — see searchBM25WithRank for rationale (567× speedup verified).
|
|
646
|
+
try {
|
|
647
|
+
const placeholders = agentIds.map(() => '?').join(',');
|
|
648
|
+
const innerLimit = Math.max(limit * 5, 50);
|
|
649
|
+
const rows = this.db.prepare(`
|
|
650
|
+
WITH top_fts AS (
|
|
651
|
+
SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
|
|
652
|
+
)
|
|
653
|
+
SELECT e.*, top_fts.rank FROM top_fts
|
|
654
|
+
JOIN engrams e ON e.rowid = top_fts.rowid
|
|
655
|
+
WHERE e.agent_id IN (${placeholders}) AND e.retracted = 0
|
|
656
|
+
ORDER BY top_fts.rank
|
|
657
|
+
LIMIT ?
|
|
658
|
+
`).all(sanitized, innerLimit, ...agentIds, limit) as any[];
|
|
659
|
+
|
|
660
|
+
return rows.map(r => ({
|
|
661
|
+
engram: this.rowToEngram(r),
|
|
662
|
+
bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
|
|
663
|
+
}));
|
|
664
|
+
} catch {
|
|
665
|
+
return [];
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/**
|
|
670
|
+
* Get all distinct agent IDs that share a workspace (requires coord_agents table).
|
|
671
|
+
* Returns just the queried agentId if coordination tables don't exist.
|
|
672
|
+
*/
|
|
673
|
+
getWorkspaceAgentIds(agentId: string, workspace: string): string[] {
|
|
674
|
+
try {
|
|
675
|
+
// Return agent names (not UUIDs) — engrams.agent_id uses name strings
|
|
676
|
+
const rows = this.db.prepare(
|
|
677
|
+
`SELECT DISTINCT name FROM coord_agents WHERE workspace = ? AND status != 'dead'`
|
|
678
|
+
).all(workspace) as Array<{ name: string }>;
|
|
679
|
+
const names = rows.map(r => r.name);
|
|
680
|
+
// Ensure the querying agent is always included
|
|
681
|
+
if (!names.includes(agentId)) names.push(agentId);
|
|
682
|
+
return names;
|
|
683
|
+
} catch {
|
|
684
|
+
// No coordination tables — fall back to single agent
|
|
685
|
+
return [agentId];
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Touch an engram: increment access count, update last_accessed, and
|
|
691
|
+
* nudge confidence upward. Each retrieval is weak evidence the memory
|
|
692
|
+
* is useful — bounded so only explicit feedback can push confidence
|
|
693
|
+
* above 0.85. Diminishing returns: first accesses matter most.
|
|
694
|
+
*
|
|
695
|
+
* Boost: +0.02 per access, scaled by 1/sqrt(accessCount+1), capped at 0.85.
|
|
696
|
+
*/
|
|
697
|
+
touchEngram(id: string): void {
|
|
698
|
+
this.db.prepare(`
|
|
699
|
+
UPDATE engrams
|
|
700
|
+
SET access_count = access_count + 1,
|
|
701
|
+
last_accessed = ?,
|
|
702
|
+
confidence = MIN(0.85, confidence + 0.02 / (1.0 + sqrt(access_count)))
|
|
703
|
+
WHERE id = ?
|
|
704
|
+
`).run(new Date().toISOString(), id);
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
updateStage(id: string, stage: EngramStage): void {
|
|
708
|
+
this.db.prepare('UPDATE engrams SET stage = ? WHERE id = ?').run(stage, id);
|
|
709
|
+
this.cacheUpdateStage(id, stage);
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Replace an engram's content. Used by the fade phase of consolidation
|
|
714
|
+
* (Paper 1: storage degradation) to coarsen un-recalled memories without
|
|
715
|
+
* deleting them — concept + tags stay intact for tag-based retrieval, but
|
|
716
|
+
* BM25 content surface area shrinks.
|
|
717
|
+
*/
|
|
718
|
+
updateContent(id: string, content: string): void {
|
|
719
|
+
this.db.prepare('UPDATE engrams SET content = ? WHERE id = ?').run(content, id);
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
updateConfidence(id: string, confidence: number): void {
|
|
723
|
+
this.db.prepare('UPDATE engrams SET confidence = ? WHERE id = ?').run(
|
|
724
|
+
Math.max(0, Math.min(1, confidence)), id
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
updateEmbedding(id: string, embedding: number[], modelId?: string): void {
|
|
729
|
+
const blob = Buffer.from(new Float32Array(embedding).buffer);
|
|
730
|
+
if (modelId) {
|
|
731
|
+
this.db.prepare('UPDATE engrams SET embedding = ?, embedding_model = ? WHERE id = ?').run(blob, modelId, id);
|
|
732
|
+
} else {
|
|
733
|
+
this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
|
|
734
|
+
}
|
|
735
|
+
this.cacheUpdateEmbedding(id, embedding);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
retractEngram(id: string, retractedBy: string | null): void {
|
|
739
|
+
this.db.prepare(`
|
|
740
|
+
UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
|
|
741
|
+
`).run(retractedBy, new Date().toISOString(), id);
|
|
742
|
+
this.cacheRetract(id);
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
deleteEngram(id: string): void {
|
|
746
|
+
this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
|
|
747
|
+
this.cacheRemove(id);
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Time warp — shift all timestamps backward by ms milliseconds.
|
|
752
|
+
* Used for testing time-dependent behavior (decay, forgetting).
|
|
753
|
+
* Returns count of records shifted.
|
|
754
|
+
*/
|
|
755
|
+
timeWarp(agentId: string, ms: number): number {
|
|
756
|
+
let count = 0;
|
|
757
|
+
const shiftSec = Math.round(ms / 1000);
|
|
758
|
+
// Shift engram timestamps
|
|
759
|
+
const r1 = this.db.prepare(`
|
|
760
|
+
UPDATE engrams SET
|
|
761
|
+
created_at = datetime(created_at, '-${shiftSec} seconds'),
|
|
762
|
+
last_accessed = datetime(last_accessed, '-${shiftSec} seconds')
|
|
763
|
+
WHERE agent_id = ?
|
|
764
|
+
`).run(agentId);
|
|
765
|
+
count += r1.changes;
|
|
766
|
+
// Shift association timestamps
|
|
767
|
+
const r2 = this.db.prepare(`
|
|
768
|
+
UPDATE associations SET
|
|
769
|
+
created_at = datetime(created_at, '-${shiftSec} seconds'),
|
|
770
|
+
last_activated = datetime(last_activated, '-${shiftSec} seconds')
|
|
771
|
+
WHERE from_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
|
|
772
|
+
OR to_engram_id IN (SELECT id FROM engrams WHERE agent_id = ?)
|
|
773
|
+
`).run(agentId, agentId);
|
|
774
|
+
count += r2.changes;
|
|
775
|
+
return count;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// --- Full-text search (BM25) ---
|
|
779
|
+
|
|
780
|
+
searchBM25(agentId: string, query: string, limit: number = 10): Engram[] {
|
|
781
|
+
return this.searchBM25WithRank(agentId, query, limit).map(r => r.engram);
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Vector similarity top-K search. SQLite has no native vector index, so we
|
|
786
|
+
* iterate over the slim cache, compute cosine similarity in-process, and
|
|
787
|
+
* return the top-K nearest neighbors. PGlite uses pgvector + ivfflat for
|
|
788
|
+
* the same operation in O(log N).
|
|
789
|
+
*
|
|
790
|
+
* Returns engrams with their cosine distance (0 = identical, 2 = opposite)
|
|
791
|
+
* to match the PGlite contract. Activation engine consumes `1 - distance`
|
|
792
|
+
* as the cosine similarity score.
|
|
793
|
+
*/
|
|
794
|
+
searchByVector(
|
|
795
|
+
agentId: string,
|
|
796
|
+
vec: number[],
|
|
797
|
+
limit: number = 10,
|
|
798
|
+
): Array<{ engram: Engram; distance: number }> {
|
|
799
|
+
if (!vec || vec.length === 0) return [];
|
|
800
|
+
// Include both 'active' and 'fading' — faded engrams retain their embedding
|
|
801
|
+
// (Paper 1: storage degradation; only content is trimmed) so they should still
|
|
802
|
+
// participate in vector recall. Concept + tags + truncated content remain in BM25.
|
|
803
|
+
const activeSlim = this.getEngramsByAgentSlim(agentId, 'active', false);
|
|
804
|
+
const fadingSlim = this.getEngramsByAgentSlim(agentId, 'fading', false);
|
|
805
|
+
const slim = activeSlim.concat(fadingSlim);
|
|
806
|
+
const scored: Array<{ id: string; sim: number }> = [];
|
|
807
|
+
for (const e of slim) {
|
|
808
|
+
if (!e.embedding) continue;
|
|
809
|
+
const sim = cosineSimilarity(vec, e.embedding);
|
|
810
|
+
scored.push({ id: e.id, sim });
|
|
811
|
+
}
|
|
812
|
+
scored.sort((a, b) => b.sim - a.sim);
|
|
813
|
+
const topIds = scored.slice(0, limit).map(s => s.id);
|
|
814
|
+
if (topIds.length === 0) return [];
|
|
815
|
+
const engrams = this.getEngramsByIds(topIds);
|
|
816
|
+
const engramMap = new Map(engrams.map(e => [e.id, e]));
|
|
817
|
+
return scored
|
|
818
|
+
.slice(0, limit)
|
|
819
|
+
.map(s => ({ engram: engramMap.get(s.id), sim: s.sim }))
|
|
820
|
+
.filter((x): x is { engram: Engram; sim: number } => x.engram !== undefined)
|
|
821
|
+
.map(x => ({ engram: x.engram, distance: 1 - x.sim }));
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* BM25 search returning rank scores alongside engrams.
|
|
826
|
+
* FTS5 rank is negative (lower = better match).
|
|
827
|
+
* We normalize to 0-1 where higher = better.
|
|
828
|
+
*/
|
|
829
|
+
searchBM25WithRank(agentId: string, query: string, limit: number = 10): { engram: Engram; bm25Score: number }[] {
|
|
830
|
+
// Sanitize query for FTS5: quote each word to prevent column name interpretation
|
|
831
|
+
const sanitized = query
|
|
832
|
+
.replace(/[^\w\s]/g, ' ') // Split on punctuation so hyphenated IDs (PROJ-1000) match FTS5's separator-split tokens.
|
|
833
|
+
.split(/\s+/)
|
|
834
|
+
.filter(w => w.length > 1)
|
|
835
|
+
.map(w => `"${w}"`)
|
|
836
|
+
.join(' OR ');
|
|
837
|
+
|
|
838
|
+
if (!sanitized) return [];
|
|
839
|
+
|
|
840
|
+
// CTE prefilter: force FTS5 to apply LIMIT before joining engrams.
|
|
841
|
+
//
|
|
842
|
+
// Why: the obvious query (JOIN engrams_fts ON rowid + WHERE MATCH + ORDER BY rank LIMIT N)
|
|
843
|
+
// makes SQLite's planner materialize ALL matching FTS rows joined with engrams
|
|
844
|
+
// before applying LIMIT. With wide OR queries on a 17K-engram index, that's
|
|
845
|
+
// thousands of row materializations including 1.5KB embedding blobs — measured
|
|
846
|
+
// at 3682ms for a 5-term OR query.
|
|
847
|
+
//
|
|
848
|
+
// The CTE forces FTS5 to LIMIT first (sub-ms), then join only the top-K rowids.
|
|
849
|
+
// Same query plan, 567× faster (3682ms → 6ms verified on 17K engrams).
|
|
850
|
+
//
|
|
851
|
+
// The inner LIMIT (limit * 5) over-fetches because the agent_id + retracted
|
|
852
|
+
// filter is applied AFTER the CTE. limit*5 gives enough headroom that filtered
|
|
853
|
+
// results still satisfy the outer LIMIT for typical workloads (single agent
|
|
854
|
+
// dominant, low retracted rate).
|
|
855
|
+
try {
|
|
856
|
+
const innerLimit = Math.max(limit * 5, 50);
|
|
857
|
+
const rows = this.db.prepare(`
|
|
858
|
+
WITH top_fts AS (
|
|
859
|
+
SELECT rowid, rank FROM engrams_fts WHERE engrams_fts MATCH ? ORDER BY rank LIMIT ?
|
|
860
|
+
)
|
|
861
|
+
SELECT e.*, top_fts.rank FROM top_fts
|
|
862
|
+
JOIN engrams e ON e.rowid = top_fts.rowid
|
|
863
|
+
WHERE e.agent_id = ? AND e.retracted = 0
|
|
864
|
+
ORDER BY top_fts.rank
|
|
865
|
+
LIMIT ?
|
|
866
|
+
`).all(sanitized, innerLimit, agentId, limit) as any[];
|
|
867
|
+
|
|
868
|
+
return rows.map(r => ({
|
|
869
|
+
engram: this.rowToEngram(r),
|
|
870
|
+
// Normalize: rank is negative, more negative = better match.
|
|
871
|
+
// |rank| / (1 + |rank|) gives 0-1 where higher = better.
|
|
872
|
+
bm25Score: Math.abs(r.rank ?? 0) / (1 + Math.abs(r.rank ?? 0)),
|
|
873
|
+
}));
|
|
874
|
+
} catch {
|
|
875
|
+
return [];
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// --- Diagnostic search (deterministic, not cognitive) ---
|
|
880
|
+
|
|
881
|
+
search(query: SearchQuery): Engram[] {
|
|
882
|
+
let sql = 'SELECT * FROM engrams WHERE agent_id = ?';
|
|
883
|
+
const params: any[] = [query.agentId];
|
|
884
|
+
|
|
885
|
+
if (query.text) {
|
|
886
|
+
sql += ' AND (content LIKE ? OR concept LIKE ?)';
|
|
887
|
+
params.push(`%${query.text}%`, `%${query.text}%`);
|
|
888
|
+
}
|
|
889
|
+
if (query.concept) {
|
|
890
|
+
sql += ' AND concept = ?';
|
|
891
|
+
params.push(query.concept);
|
|
892
|
+
}
|
|
893
|
+
if (query.stage) {
|
|
894
|
+
sql += ' AND stage = ?';
|
|
895
|
+
params.push(query.stage);
|
|
896
|
+
}
|
|
897
|
+
if (query.retracted !== undefined) {
|
|
898
|
+
sql += ' AND retracted = ?';
|
|
899
|
+
params.push(query.retracted ? 1 : 0);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// ── Tag filters (0.8 Cluster B) ──
|
|
903
|
+
// AND-set: merge legacy `tags` + new `tagsAll`. Both apply (intersection).
|
|
904
|
+
const allTags = [...(query.tags ?? []), ...(query.tagsAll ?? [])];
|
|
905
|
+
if (allTags.length > 0) {
|
|
906
|
+
for (const tag of allTags) {
|
|
907
|
+
sql += ' AND tags LIKE ?';
|
|
908
|
+
params.push(`%"${tag}"%`);
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
// OR-set: at least one must match.
|
|
912
|
+
if (query.tagsAny && query.tagsAny.length > 0) {
|
|
913
|
+
const ors = query.tagsAny.map(() => 'tags LIKE ?').join(' OR ');
|
|
914
|
+
sql += ` AND (${ors})`;
|
|
915
|
+
for (const tag of query.tagsAny) params.push(`%"${tag}"%`);
|
|
916
|
+
}
|
|
917
|
+
// NOT-set: none may match.
|
|
918
|
+
if (query.tagsNone && query.tagsNone.length > 0) {
|
|
919
|
+
const ors = query.tagsNone.map(() => 'tags LIKE ?').join(' OR ');
|
|
920
|
+
sql += ` AND NOT (${ors})`;
|
|
921
|
+
for (const tag of query.tagsNone) params.push(`%"${tag}"%`);
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
// ── Sort (0.8 Cluster B) ──
|
|
925
|
+
// Default preserves legacy behavior (last_accessed DESC). Map TS camelCase
|
|
926
|
+
// to SQL snake_case. `sequence` puts NULLs last via (col IS NULL) trick.
|
|
927
|
+
const sortCol = ({
|
|
928
|
+
createdAt: 'created_at',
|
|
929
|
+
sequence: 'sequence',
|
|
930
|
+
salience: 'salience',
|
|
931
|
+
confidence: 'confidence',
|
|
932
|
+
lastAccessed: 'last_accessed',
|
|
933
|
+
} as const)[query.sortBy ?? 'lastAccessed'];
|
|
934
|
+
const dir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
|
|
935
|
+
if (query.sortBy === 'sequence') {
|
|
936
|
+
// NULLs last regardless of direction so engrams without a story-time
|
|
937
|
+
// value don't shuffle into the middle of sequenced results.
|
|
938
|
+
sql += ` ORDER BY (sequence IS NULL), sequence ${dir}`;
|
|
939
|
+
} else {
|
|
940
|
+
sql += ` ORDER BY ${sortCol} ${dir}`;
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
sql += ` LIMIT ? OFFSET ?`;
|
|
944
|
+
params.push(query.limit ?? 50, query.offset ?? 0);
|
|
945
|
+
|
|
946
|
+
return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Get the most recently created engram for an agent (for temporal adjacency edges).
|
|
951
|
+
*/
|
|
952
|
+
getLatestEngram(agentId: string, excludeId?: string): Engram | null {
|
|
953
|
+
let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0';
|
|
954
|
+
const params: any[] = [agentId];
|
|
955
|
+
if (excludeId) {
|
|
956
|
+
sql += ' AND id != ?';
|
|
957
|
+
params.push(excludeId);
|
|
958
|
+
}
|
|
959
|
+
sql += ' ORDER BY created_at DESC LIMIT 1';
|
|
960
|
+
const row = this.db.prepare(sql).get(...params) as any;
|
|
961
|
+
return row ? this.rowToEngram(row) : null;
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
// --- Task management ---
|
|
965
|
+
|
|
966
|
+
updateTaskStatus(id: string, status: TaskStatus): void {
|
|
967
|
+
this.db.prepare('UPDATE engrams SET task_status = ? WHERE id = ?').run(status, id);
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
updateTaskPriority(id: string, priority: TaskPriority): void {
|
|
971
|
+
this.db.prepare('UPDATE engrams SET task_priority = ? WHERE id = ?').run(priority, id);
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
updateBlockedBy(id: string, blockedBy: string | null): void {
|
|
975
|
+
this.db.prepare('UPDATE engrams SET blocked_by = ?, task_status = ? WHERE id = ?')
|
|
976
|
+
.run(blockedBy, blockedBy ? 'blocked' : 'open', id);
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* Get tasks for an agent, optionally filtered by status.
|
|
981
|
+
* Results ordered by priority (urgent > high > medium > low), then creation date.
|
|
982
|
+
*/
|
|
983
|
+
getTasks(agentId: string, status?: TaskStatus): Engram[] {
|
|
984
|
+
let sql = 'SELECT * FROM engrams WHERE agent_id = ? AND task_status IS NOT NULL AND retracted = 0';
|
|
985
|
+
const params: any[] = [agentId];
|
|
986
|
+
if (status) {
|
|
987
|
+
sql += ' AND task_status = ?';
|
|
988
|
+
params.push(status);
|
|
989
|
+
}
|
|
990
|
+
sql += ` ORDER BY
|
|
991
|
+
CASE task_priority
|
|
992
|
+
WHEN 'urgent' THEN 0
|
|
993
|
+
WHEN 'high' THEN 1
|
|
994
|
+
WHEN 'medium' THEN 2
|
|
995
|
+
WHEN 'low' THEN 3
|
|
996
|
+
ELSE 4
|
|
997
|
+
END,
|
|
998
|
+
created_at DESC`;
|
|
999
|
+
return (this.db.prepare(sql).all(...params) as any[]).map(r => this.rowToEngram(r));
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* Get the next actionable task — highest priority that's not blocked or done.
|
|
1004
|
+
*/
|
|
1005
|
+
getNextTask(agentId: string): Engram | null {
|
|
1006
|
+
const row = this.db.prepare(`
|
|
1007
|
+
SELECT * FROM engrams
|
|
1008
|
+
WHERE agent_id = ? AND task_status IN ('open', 'in_progress') AND retracted = 0
|
|
1009
|
+
ORDER BY
|
|
1010
|
+
CASE task_status WHEN 'in_progress' THEN 0 ELSE 1 END,
|
|
1011
|
+
CASE task_priority
|
|
1012
|
+
WHEN 'urgent' THEN 0
|
|
1013
|
+
WHEN 'high' THEN 1
|
|
1014
|
+
WHEN 'medium' THEN 2
|
|
1015
|
+
WHEN 'low' THEN 3
|
|
1016
|
+
ELSE 4
|
|
1017
|
+
END,
|
|
1018
|
+
created_at ASC
|
|
1019
|
+
LIMIT 1
|
|
1020
|
+
`).get(agentId) as any;
|
|
1021
|
+
return row ? this.rowToEngram(row) : null;
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// --- Supersession ---
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* Mark an engram as superseded by another.
|
|
1028
|
+
* The old memory stays in the DB (historical) but gets down-ranked in recall.
|
|
1029
|
+
*/
|
|
1030
|
+
supersedeEngram(oldId: string, newId: string): void {
|
|
1031
|
+
this.db.prepare('UPDATE engrams SET superseded_by = ? WHERE id = ?').run(newId, oldId);
|
|
1032
|
+
this.db.prepare('UPDATE engrams SET supersedes = ? WHERE id = ?').run(oldId, newId);
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/**
|
|
1036
|
+
* Find the most recent active engram matching `concept` (case-insensitive,
|
|
1037
|
+
* trimmed) and all of `requiredTags`. Excludes superseded and retracted
|
|
1038
|
+
* engrams. Used by /memory/supersede Form B (0.8 Cluster D) and by
|
|
1039
|
+
* /memory/write's references[] resolution.
|
|
1040
|
+
*
|
|
1041
|
+
* Returns null if no match.
|
|
1042
|
+
*/
|
|
1043
|
+
findActiveMatchByConcept(
|
|
1044
|
+
agentId: string,
|
|
1045
|
+
concept: string,
|
|
1046
|
+
requiredTags?: string[],
|
|
1047
|
+
): Engram | null {
|
|
1048
|
+
let sql = `SELECT * FROM engrams
|
|
1049
|
+
WHERE agent_id = ?
|
|
1050
|
+
AND LOWER(TRIM(concept)) = LOWER(TRIM(?))
|
|
1051
|
+
AND stage = 'active'
|
|
1052
|
+
AND retracted = 0
|
|
1053
|
+
AND superseded_by IS NULL`;
|
|
1054
|
+
const params: any[] = [agentId, concept];
|
|
1055
|
+
if (requiredTags && requiredTags.length > 0) {
|
|
1056
|
+
for (const tag of requiredTags) {
|
|
1057
|
+
sql += ' AND tags LIKE ?';
|
|
1058
|
+
params.push(`%"${tag}"%`);
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
sql += ' ORDER BY created_at DESC LIMIT 1';
|
|
1062
|
+
const row = this.db.prepare(sql).get(...params) as any;
|
|
1063
|
+
return row ? this.rowToEngram(row) : null;
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* Check if an engram has been superseded.
|
|
1068
|
+
*/
|
|
1069
|
+
isSuperseded(id: string): boolean {
|
|
1070
|
+
const row = this.db.prepare('SELECT superseded_by FROM engrams WHERE id = ?').get(id) as any;
|
|
1071
|
+
return row?.superseded_by != null;
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
updateMemoryClass(id: string, memoryClass: MemoryClass): void {
|
|
1075
|
+
this.db.prepare('UPDATE engrams SET memory_class = ? WHERE id = ?').run(memoryClass, id);
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
updateTags(id: string, tags: string[]): void {
|
|
1079
|
+
this.db.prepare('UPDATE engrams SET tags = ? WHERE id = ?').run(JSON.stringify(tags), id);
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
// --- Associations ---
|
|
1083
|
+
|
|
1084
|
+
upsertAssociation(
|
|
1085
|
+
fromId: string, toId: string, weight: number,
|
|
1086
|
+
type: AssociationType = 'hebbian', confidence: number = 0.5
|
|
1087
|
+
): Association {
|
|
1088
|
+
const now = new Date().toISOString();
|
|
1089
|
+
const id = randomUUID();
|
|
1090
|
+
|
|
1091
|
+
this.db.prepare(`
|
|
1092
|
+
INSERT INTO associations (id, from_engram_id, to_engram_id, weight, confidence, type, activation_count, created_at, last_activated)
|
|
1093
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)
|
|
1094
|
+
ON CONFLICT(from_engram_id, to_engram_id) DO UPDATE SET
|
|
1095
|
+
weight = ?, confidence = ?, last_activated = ?, activation_count = activation_count + 1
|
|
1096
|
+
`).run(id, fromId, toId, weight, confidence, type, now, now, weight, confidence, now);
|
|
1097
|
+
|
|
1098
|
+
return this.getAssociation(fromId, toId)!;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
getAssociation(fromId: string, toId: string): Association | null {
|
|
1102
|
+
const row = this.db.prepare(
|
|
1103
|
+
'SELECT * FROM associations WHERE from_engram_id = ? AND to_engram_id = ?'
|
|
1104
|
+
).get(fromId, toId) as any;
|
|
1105
|
+
return row ? this.rowToAssociation(row) : null;
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
getAssociationsFor(engramId: string): Association[] {
|
|
1109
|
+
const rows = this.db.prepare(
|
|
1110
|
+
'SELECT * FROM associations WHERE from_engram_id = ? OR to_engram_id = ?'
|
|
1111
|
+
).all(engramId, engramId);
|
|
1112
|
+
return (rows as any[]).map(r => this.rowToAssociation(r));
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* Aggregate association stats per engram — count of edges and sum of weights.
|
|
1117
|
+
*
|
|
1118
|
+
* Why: the activation scoring loop only uses `associations.length` (Hebbian gate)
|
|
1119
|
+
* and `sum of weights` (Hebbian mean + centrality). It doesn't read individual
|
|
1120
|
+
* association fields. Returning scalar stats avoids materializing thousands of
|
|
1121
|
+
* Association objects.
|
|
1122
|
+
*
|
|
1123
|
+
* Phase-breakdown spike (2026-05-08, post-0.7.10) showed
|
|
1124
|
+
* `getAssociationsForBatch` over ~300 survivors took 222ms (25% of recall floor).
|
|
1125
|
+
* Stats-only aggregate via a single GROUP BY drops this to ~20ms.
|
|
1126
|
+
*
|
|
1127
|
+
* Graph walk still needs full Association rows, but it operates on top-N
|
|
1128
|
+
* (~30 candidates) — its per-call `getAssociationsFor` is cheap.
|
|
1129
|
+
*/
|
|
1130
|
+
getAssociationStatsForBatch(engramIds: string[]): Map<string, { count: number; sumWeight: number }> {
|
|
1131
|
+
const result = new Map<string, { count: number; sumWeight: number }>();
|
|
1132
|
+
if (engramIds.length === 0) return result;
|
|
1133
|
+
|
|
1134
|
+
const CHUNK = 400;
|
|
1135
|
+
for (let i = 0; i < engramIds.length; i += CHUNK) {
|
|
1136
|
+
const chunk = engramIds.slice(i, i + CHUNK);
|
|
1137
|
+
const placeholders = chunk.map(() => '?').join(',');
|
|
1138
|
+
// UNION ALL counts each association once per endpoint that's in the candidate
|
|
1139
|
+
// set — same semantics as the existing getAssociationsForBatch which buckets
|
|
1140
|
+
// associations under both their from and to engram. Self-loops would be
|
|
1141
|
+
// double-counted, but they're rare in practice and the prior code handled them
|
|
1142
|
+
// identically.
|
|
1143
|
+
const rows = this.db.prepare(
|
|
1144
|
+
`SELECT id, SUM(cnt) AS count, SUM(sw) AS sum_weight FROM (
|
|
1145
|
+
SELECT from_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE from_engram_id IN (${placeholders})
|
|
1146
|
+
UNION ALL
|
|
1147
|
+
SELECT to_engram_id AS id, 1 AS cnt, weight AS sw FROM associations WHERE to_engram_id IN (${placeholders})
|
|
1148
|
+
)
|
|
1149
|
+
WHERE id IN (${placeholders})
|
|
1150
|
+
GROUP BY id`
|
|
1151
|
+
).all(...chunk, ...chunk, ...chunk) as Array<{ id: string; count: number; sum_weight: number }>;
|
|
1152
|
+
|
|
1153
|
+
for (const r of rows) {
|
|
1154
|
+
result.set(r.id, { count: r.count, sumWeight: r.sum_weight });
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
// Ensure every requested id has an entry (even zero-edge engrams)
|
|
1158
|
+
for (const id of engramIds) {
|
|
1159
|
+
if (!result.has(id)) result.set(id, { count: 0, sumWeight: 0 });
|
|
1160
|
+
}
|
|
1161
|
+
return result;
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/**
|
|
1165
|
+
* Batch variant of getAssociationsFor — fetches associations for many engrams
|
|
1166
|
+
* in a single query, returning a Map keyed by engram id.
|
|
1167
|
+
*
|
|
1168
|
+
* Why: per-candidate `getAssociationsFor` calls inside the activation scoring
|
|
1169
|
+
* loop are an N+1. Measured at 1300ms for 10K candidates (sub-ms per call but
|
|
1170
|
+
* accumulating). One IN-clause query reduces this to ~50ms.
|
|
1171
|
+
*/
|
|
1172
|
+
getAssociationsForBatch(engramIds: string[]): Map<string, Association[]> {
|
|
1173
|
+
const result = new Map<string, Association[]>();
|
|
1174
|
+
if (engramIds.length === 0) return result;
|
|
1175
|
+
|
|
1176
|
+
// SQLite's default SQLITE_LIMIT_VARIABLE_NUMBER is 999. Chunk to stay safely below.
|
|
1177
|
+
// We bind each id twice (from + to), so chunks of 400 use 800 placeholders.
|
|
1178
|
+
const CHUNK = 400;
|
|
1179
|
+
for (let i = 0; i < engramIds.length; i += CHUNK) {
|
|
1180
|
+
const chunk = engramIds.slice(i, i + CHUNK);
|
|
1181
|
+
const placeholders = chunk.map(() => '?').join(',');
|
|
1182
|
+
const rows = this.db.prepare(
|
|
1183
|
+
`SELECT * FROM associations
|
|
1184
|
+
WHERE from_engram_id IN (${placeholders}) OR to_engram_id IN (${placeholders})`
|
|
1185
|
+
).all(...chunk, ...chunk) as any[];
|
|
1186
|
+
for (const r of rows) {
|
|
1187
|
+
const a = this.rowToAssociation(r);
|
|
1188
|
+
// Bucket by both endpoints — getAssociationsFor returns either-direction matches.
|
|
1189
|
+
const fromList = result.get(a.fromEngramId) ?? [];
|
|
1190
|
+
fromList.push(a);
|
|
1191
|
+
result.set(a.fromEngramId, fromList);
|
|
1192
|
+
if (a.toEngramId !== a.fromEngramId) {
|
|
1193
|
+
const toList = result.get(a.toEngramId) ?? [];
|
|
1194
|
+
toList.push(a);
|
|
1195
|
+
result.set(a.toEngramId, toList);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
// Ensure every requested id has an entry (even if empty) so callers can
|
|
1200
|
+
// .get() without null-checking.
|
|
1201
|
+
for (const id of engramIds) {
|
|
1202
|
+
if (!result.has(id)) result.set(id, []);
|
|
1203
|
+
}
|
|
1204
|
+
return result;
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
getOutgoingAssociations(engramId: string): Association[] {
|
|
1208
|
+
const rows = this.db.prepare(
|
|
1209
|
+
'SELECT * FROM associations WHERE from_engram_id = ?'
|
|
1210
|
+
).all(engramId);
|
|
1211
|
+
return (rows as any[]).map(r => this.rowToAssociation(r));
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
countAssociationsFor(engramId: string): number {
|
|
1215
|
+
const row = this.db.prepare(
|
|
1216
|
+
'SELECT COUNT(*) as count FROM associations WHERE from_engram_id = ?'
|
|
1217
|
+
).get(engramId) as any;
|
|
1218
|
+
return row.count;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
getWeakestAssociation(engramId: string): Association | null {
|
|
1222
|
+
const row = this.db.prepare(
|
|
1223
|
+
'SELECT * FROM associations WHERE from_engram_id = ? ORDER BY weight ASC LIMIT 1'
|
|
1224
|
+
).get(engramId) as any;
|
|
1225
|
+
return row ? this.rowToAssociation(row) : null;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
deleteAssociation(id: string): void {
|
|
1229
|
+
this.db.prepare('DELETE FROM associations WHERE id = ?').run(id);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
getAllAssociations(agentId: string): Association[] {
|
|
1233
|
+
const rows = this.db.prepare(`
|
|
1234
|
+
SELECT a.* FROM associations a
|
|
1235
|
+
JOIN engrams e ON a.from_engram_id = e.id
|
|
1236
|
+
WHERE e.agent_id = ?
|
|
1237
|
+
`).all(agentId);
|
|
1238
|
+
return (rows as any[]).map(r => this.rowToAssociation(r));
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
// --- Eviction ---
|
|
1242
|
+
|
|
1243
|
+
getEvictionCandidates(agentId: string, limit: number): Engram[] {
|
|
1244
|
+
// Lowest combined score: low salience + low access + low confidence + oldest
|
|
1245
|
+
const rows = this.db.prepare(`
|
|
1246
|
+
SELECT * FROM engrams
|
|
1247
|
+
WHERE agent_id = ? AND stage = 'active' AND retracted = 0
|
|
1248
|
+
ORDER BY (salience * 0.3 + confidence * 0.3 + (CAST(access_count AS REAL) / (access_count + 5)) * 0.2 +
|
|
1249
|
+
(1.0 / (1.0 + (julianday('now') - julianday(last_accessed)))) * 0.2) ASC
|
|
1250
|
+
LIMIT ?
|
|
1251
|
+
`).all(agentId, limit) as any[];
|
|
1252
|
+
return rows.map(r => this.rowToEngram(r));
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
getActiveCount(agentId: string): number {
|
|
1256
|
+
const row = this.db.prepare(
|
|
1257
|
+
"SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'active'"
|
|
1258
|
+
).get(agentId) as any;
|
|
1259
|
+
return row.count;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
getStagingCount(agentId: string): number {
|
|
1263
|
+
const row = this.db.prepare(
|
|
1264
|
+
"SELECT COUNT(*) as count FROM engrams WHERE agent_id = ? AND stage = 'staging'"
|
|
1265
|
+
).get(agentId) as any;
|
|
1266
|
+
return row.count;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// --- Staging buffer ---
|
|
1270
|
+
|
|
1271
|
+
getExpiredStaging(): Engram[] {
|
|
1272
|
+
const now = Date.now();
|
|
1273
|
+
const rows = this.db.prepare(`
|
|
1274
|
+
SELECT * FROM engrams WHERE stage = 'staging' AND ttl IS NOT NULL
|
|
1275
|
+
`).all() as any[];
|
|
1276
|
+
|
|
1277
|
+
return rows
|
|
1278
|
+
.map(r => this.rowToEngram(r))
|
|
1279
|
+
.filter(e => e.ttl && (e.createdAt.getTime() + e.ttl) < now);
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
// --- Eval event logging ---
|
|
1283
|
+
|
|
1284
|
+
logActivationEvent(event: ActivationEvent): void {
|
|
1285
|
+
this.db.prepare(`
|
|
1286
|
+
INSERT INTO activation_events (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
|
|
1287
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1288
|
+
`).run(
|
|
1289
|
+
event.id, event.agentId, event.timestamp.toISOString(),
|
|
1290
|
+
event.context, event.resultsReturned, event.topScore,
|
|
1291
|
+
event.latencyMs, JSON.stringify(event.engramIds)
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
logStagingEvent(event: StagingEvent): void {
|
|
1296
|
+
this.db.prepare(`
|
|
1297
|
+
INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
|
|
1298
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1299
|
+
`).run(
|
|
1300
|
+
event.engramId, event.agentId, event.action,
|
|
1301
|
+
event.resonanceScore, event.timestamp.toISOString(), event.ageMs
|
|
1302
|
+
);
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
logRetrievalFeedback(activationEventId: string | null, engramId: string, useful: boolean, context: string): void {
|
|
1306
|
+
this.db.prepare(`
|
|
1307
|
+
INSERT INTO retrieval_feedback (id, activation_event_id, engram_id, useful, context, timestamp)
|
|
1308
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1309
|
+
`).run(randomUUID(), activationEventId, engramId, useful ? 1 : 0, context, new Date().toISOString());
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
// --- Eval metrics queries ---
|
|
1313
|
+
|
|
1314
|
+
getRetrievalPrecision(agentId: string, windowHours: number = 24): number {
|
|
1315
|
+
const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
|
|
1316
|
+
const row = this.db.prepare(`
|
|
1317
|
+
SELECT
|
|
1318
|
+
COUNT(CASE WHEN useful = 1 THEN 1 END) as useful_count,
|
|
1319
|
+
COUNT(*) as total_count
|
|
1320
|
+
FROM retrieval_feedback rf
|
|
1321
|
+
LEFT JOIN activation_events ae ON rf.activation_event_id = ae.id
|
|
1322
|
+
JOIN engrams e ON rf.engram_id = e.id
|
|
1323
|
+
WHERE e.agent_id = ? AND rf.timestamp > ?
|
|
1324
|
+
`).get(agentId, since) as any;
|
|
1325
|
+
|
|
1326
|
+
return row.total_count > 0 ? row.useful_count / row.total_count : 0;
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
getStagingMetrics(agentId: string): { promoted: number; discarded: number; expired: number } {
|
|
1330
|
+
const row = this.db.prepare(`
|
|
1331
|
+
SELECT
|
|
1332
|
+
COUNT(CASE WHEN action = 'promoted' THEN 1 END) as promoted,
|
|
1333
|
+
COUNT(CASE WHEN action = 'discarded' THEN 1 END) as discarded,
|
|
1334
|
+
COUNT(CASE WHEN action = 'expired' THEN 1 END) as expired
|
|
1335
|
+
FROM staging_events WHERE agent_id = ?
|
|
1336
|
+
`).get(agentId) as any;
|
|
1337
|
+
return { promoted: row.promoted, discarded: row.discarded, expired: row.expired };
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
getActivationStats(agentId: string, windowHours: number = 24): {
|
|
1341
|
+
count: number; avgLatencyMs: number; p95LatencyMs: number;
|
|
1342
|
+
} {
|
|
1343
|
+
const since = new Date(Date.now() - windowHours * 3600_000).toISOString();
|
|
1344
|
+
const rows = this.db.prepare(`
|
|
1345
|
+
SELECT latency_ms FROM activation_events
|
|
1346
|
+
WHERE agent_id = ? AND timestamp > ?
|
|
1347
|
+
ORDER BY latency_ms ASC
|
|
1348
|
+
`).all(agentId, since) as { latency_ms: number }[];
|
|
1349
|
+
|
|
1350
|
+
if (rows.length === 0) return { count: 0, avgLatencyMs: 0, p95LatencyMs: 0 };
|
|
1351
|
+
|
|
1352
|
+
const total = rows.reduce((s, r) => s + r.latency_ms, 0);
|
|
1353
|
+
const p95Index = Math.min(Math.floor(rows.length * 0.95), rows.length - 1);
|
|
1354
|
+
return {
|
|
1355
|
+
count: rows.length,
|
|
1356
|
+
avgLatencyMs: total / rows.length,
|
|
1357
|
+
p95LatencyMs: rows[p95Index].latency_ms,
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
getConsolidatedCount(agentId: string): number {
|
|
1362
|
+
const row = this.db.prepare(
|
|
1363
|
+
`SELECT COUNT(*) as cnt FROM engrams WHERE agent_id = ? AND stage = 'consolidated'`
|
|
1364
|
+
).get(agentId) as any;
|
|
1365
|
+
return row.cnt;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
// --- Helpers ---
|
|
1369
|
+
|
|
1370
|
+
private rowToEngram(row: any): Engram {
|
|
1371
|
+
return {
|
|
1372
|
+
id: row.id,
|
|
1373
|
+
agentId: row.agent_id,
|
|
1374
|
+
concept: row.concept,
|
|
1375
|
+
content: row.content,
|
|
1376
|
+
embedding: row.embedding
|
|
1377
|
+
? Array.from(bufferToFloat32Array(row.embedding))
|
|
1378
|
+
: null,
|
|
1379
|
+
confidence: row.confidence,
|
|
1380
|
+
salience: row.salience,
|
|
1381
|
+
accessCount: row.access_count,
|
|
1382
|
+
lastAccessed: new Date(row.last_accessed),
|
|
1383
|
+
createdAt: new Date(row.created_at),
|
|
1384
|
+
salienceFeatures: JSON.parse(row.salience_features || '{}'),
|
|
1385
|
+
reasonCodes: JSON.parse(row.reason_codes || '[]'),
|
|
1386
|
+
stage: row.stage as EngramStage,
|
|
1387
|
+
ttl: row.ttl,
|
|
1388
|
+
retracted: !!row.retracted,
|
|
1389
|
+
retractedBy: row.retracted_by,
|
|
1390
|
+
retractedAt: row.retracted_at ? new Date(row.retracted_at) : null,
|
|
1391
|
+
tags: JSON.parse(row.tags),
|
|
1392
|
+
episodeId: row.episode_id ?? null,
|
|
1393
|
+
memoryClass: (row.memory_class ?? 'working') as MemoryClass,
|
|
1394
|
+
memoryType: (row.memory_type ?? 'unclassified') as MemoryType,
|
|
1395
|
+
supersededBy: row.superseded_by ?? null,
|
|
1396
|
+
supersedes: row.supersedes ?? null,
|
|
1397
|
+
taskStatus: row.task_status ?? null,
|
|
1398
|
+
taskPriority: row.task_priority ?? null,
|
|
1399
|
+
blockedBy: row.blocked_by ?? null,
|
|
1400
|
+
sequence: row.sequence == null ? null : Number(row.sequence),
|
|
1401
|
+
references: row.references_json ? JSON.parse(row.references_json) : null,
|
|
1402
|
+
originClass: row.origin_class ?? null,
|
|
1403
|
+
writerSession: row.writer_session ?? null,
|
|
1404
|
+
recipeId: row.recipe_id ?? null,
|
|
1405
|
+
validFrom: row.valid_from ?? null,
|
|
1406
|
+
validTo: row.valid_to ?? null,
|
|
1407
|
+
};
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
private rowToAssociation(row: any): Association {
|
|
1411
|
+
return {
|
|
1412
|
+
id: row.id,
|
|
1413
|
+
fromEngramId: row.from_engram_id,
|
|
1414
|
+
toEngramId: row.to_engram_id,
|
|
1415
|
+
weight: row.weight,
|
|
1416
|
+
confidence: row.confidence ?? 0.5,
|
|
1417
|
+
type: row.type as AssociationType,
|
|
1418
|
+
activationCount: row.activation_count ?? 0,
|
|
1419
|
+
createdAt: new Date(row.created_at),
|
|
1420
|
+
lastActivated: new Date(row.last_activated),
|
|
1421
|
+
};
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// --- Episodes ---
|
|
1425
|
+
|
|
1426
|
+
createEpisode(input: { agentId: string; label: string; embedding?: number[] }): Episode {
|
|
1427
|
+
const now = new Date().toISOString();
|
|
1428
|
+
const id = randomUUID();
|
|
1429
|
+
const embeddingBlob = input.embedding
|
|
1430
|
+
? Buffer.from(new Float32Array(input.embedding).buffer)
|
|
1431
|
+
: null;
|
|
1432
|
+
|
|
1433
|
+
this.db.prepare(`
|
|
1434
|
+
INSERT INTO episodes (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
|
|
1435
|
+
VALUES (?, ?, ?, ?, 0, ?, ?, ?)
|
|
1436
|
+
`).run(id, input.agentId, input.label, embeddingBlob, now, now, now);
|
|
1437
|
+
|
|
1438
|
+
return this.getEpisode(id)!;
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
getEpisode(id: string): Episode | null {
|
|
1442
|
+
const row = this.db.prepare('SELECT * FROM episodes WHERE id = ?').get(id) as any;
|
|
1443
|
+
return row ? this.rowToEpisode(row) : null;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
getEpisodesByAgent(agentId: string): Episode[] {
|
|
1447
|
+
const rows = this.db.prepare(
|
|
1448
|
+
'SELECT * FROM episodes WHERE agent_id = ? ORDER BY end_time DESC'
|
|
1449
|
+
).all(agentId) as any[];
|
|
1450
|
+
return rows.map(r => this.rowToEpisode(r));
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
getActiveEpisode(agentId: string, windowMs: number = 3600_000): Episode | null {
|
|
1454
|
+
// Find most recent episode that ended within the time window
|
|
1455
|
+
const cutoff = new Date(Date.now() - windowMs).toISOString();
|
|
1456
|
+
const row = this.db.prepare(`
|
|
1457
|
+
SELECT * FROM episodes WHERE agent_id = ? AND end_time > ?
|
|
1458
|
+
ORDER BY end_time DESC LIMIT 1
|
|
1459
|
+
`).get(agentId, cutoff) as any;
|
|
1460
|
+
return row ? this.rowToEpisode(row) : null;
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
addEngramToEpisode(engramId: string, episodeId: string): void {
|
|
1464
|
+
this.db.prepare('UPDATE engrams SET episode_id = ? WHERE id = ?').run(episodeId, engramId);
|
|
1465
|
+
this.db.prepare(`
|
|
1466
|
+
UPDATE episodes SET
|
|
1467
|
+
engram_count = engram_count + 1,
|
|
1468
|
+
end_time = MAX(end_time, ?)
|
|
1469
|
+
WHERE id = ?
|
|
1470
|
+
`).run(new Date().toISOString(), episodeId);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
getEngramsByEpisode(episodeId: string): Engram[] {
|
|
1474
|
+
const rows = this.db.prepare(
|
|
1475
|
+
'SELECT * FROM engrams WHERE episode_id = ? AND retracted = 0 ORDER BY created_at ASC'
|
|
1476
|
+
).all(episodeId) as any[];
|
|
1477
|
+
return rows.map(r => this.rowToEngram(r));
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
updateEpisodeEmbedding(id: string, embedding: number[]): void {
|
|
1481
|
+
const blob = Buffer.from(new Float32Array(embedding).buffer);
|
|
1482
|
+
this.db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(blob, id);
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
getEpisodeCount(agentId: string): number {
|
|
1486
|
+
const row = this.db.prepare(
|
|
1487
|
+
'SELECT COUNT(*) as cnt FROM episodes WHERE agent_id = ?'
|
|
1488
|
+
).get(agentId) as any;
|
|
1489
|
+
return row.cnt;
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
private rowToEpisode(row: any): Episode {
|
|
1493
|
+
return {
|
|
1494
|
+
id: row.id,
|
|
1495
|
+
agentId: row.agent_id,
|
|
1496
|
+
label: row.label,
|
|
1497
|
+
embedding: row.embedding
|
|
1498
|
+
? Array.from(bufferToFloat32Array(row.embedding))
|
|
1499
|
+
: null,
|
|
1500
|
+
engramCount: row.engram_count,
|
|
1501
|
+
startTime: new Date(row.start_time),
|
|
1502
|
+
endTime: new Date(row.end_time),
|
|
1503
|
+
createdAt: new Date(row.created_at),
|
|
1504
|
+
};
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
/**
|
|
1508
|
+
* Find engrams whose tags contain any of the given tag values.
|
|
1509
|
+
* Used for entity-bridge retrieval: given entity tags from top results,
|
|
1510
|
+
* find other engrams mentioning the same entities.
|
|
1511
|
+
*/
|
|
1512
|
+
findEngramsByTags(agentId: string, tags: string[], excludeIds?: Set<string>): Engram[] {
|
|
1513
|
+
if (tags.length === 0) return [];
|
|
1514
|
+
|
|
1515
|
+
// Build OR conditions for tag matching
|
|
1516
|
+
const conditions = tags.map(() => 'tags LIKE ?').join(' OR ');
|
|
1517
|
+
const params: any[] = [agentId, ...tags.map(t => `%"${t}"%`)];
|
|
1518
|
+
|
|
1519
|
+
let sql = `SELECT * FROM engrams WHERE agent_id = ? AND retracted = 0 AND (${conditions})`;
|
|
1520
|
+
const rows = this.db.prepare(sql).all(...params) as any[];
|
|
1521
|
+
|
|
1522
|
+
const results = rows.map(r => this.rowToEngram(r));
|
|
1523
|
+
if (excludeIds) {
|
|
1524
|
+
return results.filter(e => !excludeIds.has(e.id));
|
|
1525
|
+
}
|
|
1526
|
+
return results;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
// --- Checkpointing ---
|
|
1530
|
+
|
|
1531
|
+
updateAutoCheckpointWrite(agentId: string, engramId: string): void {
|
|
1532
|
+
const now = new Date().toISOString();
|
|
1533
|
+
this.db.prepare(`
|
|
1534
|
+
INSERT INTO conscious_state (agent_id, last_write_id, last_activity_at, write_count_since_consolidation, updated_at)
|
|
1535
|
+
VALUES (?, ?, ?, 1, ?)
|
|
1536
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
1537
|
+
last_write_id = excluded.last_write_id,
|
|
1538
|
+
last_activity_at = excluded.last_activity_at,
|
|
1539
|
+
write_count_since_consolidation = write_count_since_consolidation + 1,
|
|
1540
|
+
updated_at = excluded.updated_at
|
|
1541
|
+
`).run(agentId, engramId, now, now);
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
updateAutoCheckpointRecall(agentId: string, context: string, engramIds: string[]): void {
|
|
1545
|
+
const now = new Date().toISOString();
|
|
1546
|
+
this.db.prepare(`
|
|
1547
|
+
INSERT INTO conscious_state (agent_id, last_recall_context, last_recall_ids, last_activity_at, recall_count_since_consolidation, updated_at)
|
|
1548
|
+
VALUES (?, ?, ?, ?, 1, ?)
|
|
1549
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
1550
|
+
last_recall_context = excluded.last_recall_context,
|
|
1551
|
+
last_recall_ids = excluded.last_recall_ids,
|
|
1552
|
+
last_activity_at = excluded.last_activity_at,
|
|
1553
|
+
recall_count_since_consolidation = recall_count_since_consolidation + 1,
|
|
1554
|
+
updated_at = excluded.updated_at
|
|
1555
|
+
`).run(agentId, context, JSON.stringify(engramIds), now, now);
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
touchActivity(agentId: string): void {
|
|
1559
|
+
const now = new Date().toISOString();
|
|
1560
|
+
this.db.prepare(`
|
|
1561
|
+
INSERT INTO conscious_state (agent_id, last_activity_at, updated_at)
|
|
1562
|
+
VALUES (?, ?, ?)
|
|
1563
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
1564
|
+
last_activity_at = excluded.last_activity_at,
|
|
1565
|
+
updated_at = excluded.updated_at
|
|
1566
|
+
`).run(agentId, now, now);
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
saveCheckpoint(agentId: string, state: ConsciousState): void {
|
|
1570
|
+
const now = new Date().toISOString();
|
|
1571
|
+
this.db.prepare(`
|
|
1572
|
+
INSERT INTO conscious_state (agent_id, execution_state, checkpoint_at, last_activity_at, updated_at)
|
|
1573
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1574
|
+
ON CONFLICT(agent_id) DO UPDATE SET
|
|
1575
|
+
execution_state = excluded.execution_state,
|
|
1576
|
+
checkpoint_at = excluded.checkpoint_at,
|
|
1577
|
+
last_activity_at = excluded.last_activity_at,
|
|
1578
|
+
updated_at = excluded.updated_at
|
|
1579
|
+
`).run(agentId, JSON.stringify(state), now, now, now);
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
/** Record entity mentions for an engram (D9 inverted index; idempotent). */
|
|
1583
|
+
recordEntityMentions(engramId: string, agentId: string, entities: string[]): void {
|
|
1584
|
+
if (!entities || entities.length === 0) return;
|
|
1585
|
+
const stmt = this.db.prepare('INSERT OR IGNORE INTO entity_mentions (entity, engram_id, agent_id) VALUES (?, ?, ?)');
|
|
1586
|
+
for (const entity of entities) stmt.run(entity, engramId, agentId);
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
/** Indexed entities whose normalized key:value — or a registered alias — contains the term
|
|
1590
|
+
* (D11 query-side lookup). Alias matches resolve to their target entity. */
|
|
1591
|
+
searchEntities(term: string, limit: number = 8): string[] {
|
|
1592
|
+
const t = term.toLowerCase().replace(/[%_]/g, '');
|
|
1593
|
+
if (!t) return [];
|
|
1594
|
+
const rows = this.db.prepare(`
|
|
1595
|
+
SELECT DISTINCT entity FROM (
|
|
1596
|
+
SELECT entity FROM entity_mentions WHERE entity LIKE ?
|
|
1597
|
+
UNION SELECT entity FROM entity_aliases WHERE alias LIKE ?
|
|
1598
|
+
) LIMIT ?`).all(`%${t}%`, `%${t}%`, limit) as Array<{ entity: string }>;
|
|
1599
|
+
return rows.map(r => r.entity);
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
/** Engram ids mentioning an entity (alias-resolved), optionally agent-scoped. */
|
|
1603
|
+
getEngramIdsByEntity(entity: string, agentId?: string): string[] {
|
|
1604
|
+
const resolved = (this.db.prepare('SELECT entity FROM entity_aliases WHERE alias = ?')
|
|
1605
|
+
.get(entity.toLowerCase()) as { entity: string } | undefined)?.entity ?? entity.toLowerCase();
|
|
1606
|
+
const rows = agentId
|
|
1607
|
+
? this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ? AND agent_id = ?').all(resolved, agentId)
|
|
1608
|
+
: this.db.prepare('SELECT engram_id FROM entity_mentions WHERE entity = ?').all(resolved);
|
|
1609
|
+
return (rows as Array<{ engram_id: string }>).map(r => r.engram_id);
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
/** Distinct agent ids present in this store (D3 whoami; includes all stages). */
|
|
1613
|
+
listAgentIds(): string[] {
|
|
1614
|
+
const rows = this.db.prepare('SELECT DISTINCT agent_id FROM engrams ORDER BY agent_id').all() as Array<{ agent_id: string }>;
|
|
1615
|
+
return rows.map(r => r.agent_id);
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
getCheckpoint(agentId: string): CheckpointRow | null {
|
|
1619
|
+
const row = this.db.prepare('SELECT * FROM conscious_state WHERE agent_id = ?').get(agentId) as any;
|
|
1620
|
+
if (!row) return null;
|
|
1621
|
+
|
|
1622
|
+
return {
|
|
1623
|
+
agentId: row.agent_id,
|
|
1624
|
+
auto: {
|
|
1625
|
+
lastWriteId: row.last_write_id ?? null,
|
|
1626
|
+
lastRecallContext: row.last_recall_context ?? null,
|
|
1627
|
+
lastRecallIds: JSON.parse(row.last_recall_ids || '[]'),
|
|
1628
|
+
lastActivityAt: new Date(row.last_activity_at),
|
|
1629
|
+
writeCountSinceConsolidation: row.write_count_since_consolidation,
|
|
1630
|
+
recallCountSinceConsolidation: row.recall_count_since_consolidation,
|
|
1631
|
+
},
|
|
1632
|
+
executionState: row.execution_state ? JSON.parse(row.execution_state) : null,
|
|
1633
|
+
checkpointAt: row.checkpoint_at ? new Date(row.checkpoint_at) : null,
|
|
1634
|
+
lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
|
|
1635
|
+
lastMiniConsolidationAt: row.last_mini_consolidation_at ? new Date(row.last_mini_consolidation_at) : null,
|
|
1636
|
+
updatedAt: new Date(row.updated_at),
|
|
1637
|
+
};
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
markConsolidation(agentId: string, mini: boolean): void {
|
|
1641
|
+
const now = new Date().toISOString();
|
|
1642
|
+
if (mini) {
|
|
1643
|
+
this.db.prepare(`
|
|
1644
|
+
UPDATE conscious_state SET last_mini_consolidation_at = ?, updated_at = ? WHERE agent_id = ?
|
|
1645
|
+
`).run(now, now, agentId);
|
|
1646
|
+
} else {
|
|
1647
|
+
this.db.prepare(`
|
|
1648
|
+
UPDATE conscious_state SET
|
|
1649
|
+
last_consolidation_at = ?,
|
|
1650
|
+
last_mini_consolidation_at = ?,
|
|
1651
|
+
write_count_since_consolidation = 0,
|
|
1652
|
+
recall_count_since_consolidation = 0,
|
|
1653
|
+
consolidation_cycle_count = consolidation_cycle_count + 1,
|
|
1654
|
+
updated_at = ?
|
|
1655
|
+
WHERE agent_id = ?
|
|
1656
|
+
`).run(now, now, now, agentId);
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
getActiveAgents(): Array<{ agentId: string; lastActivityAt: Date; writeCount: number; recallCount: number; lastConsolidationAt: Date | null }> {
|
|
1661
|
+
const rows = this.db.prepare('SELECT * FROM conscious_state').all() as any[];
|
|
1662
|
+
return rows.map(row => ({
|
|
1663
|
+
agentId: row.agent_id,
|
|
1664
|
+
lastActivityAt: new Date(row.last_activity_at),
|
|
1665
|
+
writeCount: row.write_count_since_consolidation,
|
|
1666
|
+
recallCount: row.recall_count_since_consolidation,
|
|
1667
|
+
lastConsolidationAt: row.last_consolidation_at ? new Date(row.last_consolidation_at) : null,
|
|
1668
|
+
}));
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
getConsolidationCycleCount(agentId: string): number {
|
|
1672
|
+
const row = this.db.prepare(
|
|
1673
|
+
'SELECT consolidation_cycle_count FROM conscious_state WHERE agent_id = ?',
|
|
1674
|
+
).get(agentId) as { consolidation_cycle_count: number } | undefined;
|
|
1675
|
+
return row?.consolidation_cycle_count ?? 0;
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
close(): void {
|
|
1679
|
+
this.stopWalCheckpointTimer();
|
|
1680
|
+
this.walCheckpoint();
|
|
1681
|
+
this.db.close();
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
/**
|
|
1685
|
+
* Run `fn` in a SQL transaction. All writes succeed atomically or roll back
|
|
1686
|
+
* on throw. Synchronous (matches better-sqlite3 semantics).
|
|
1687
|
+
*
|
|
1688
|
+
* Used by 0.8 Cluster D's `/memory/supersede` Form B to wrap find-match +
|
|
1689
|
+
* performWrite + supersede + causal-edge + confidence-decay so callers
|
|
1690
|
+
* never observe an intermediate state where new engram exists but old
|
|
1691
|
+
* isn't yet superseded.
|
|
1692
|
+
*/
|
|
1693
|
+
transaction<T>(fn: () => T): T {
|
|
1694
|
+
return this.db.transaction(fn)();
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* Async-aware transaction wrapper. `fn` may await — we hold the SQLite
|
|
1699
|
+
* lock across awaits by issuing raw BEGIN/COMMIT/ROLLBACK statements rather
|
|
1700
|
+
* than using better-sqlite3's sync `db.transaction()`.
|
|
1701
|
+
*
|
|
1702
|
+
* Used by Form B (atomic write + supersede) where the body calls
|
|
1703
|
+
* async store methods through the IEngramStore contract.
|
|
1704
|
+
*/
|
|
1705
|
+
async withTransaction<T>(fn: () => Promise<T>): Promise<T> {
|
|
1706
|
+
this.db.exec('BEGIN');
|
|
1707
|
+
try {
|
|
1708
|
+
const result = await fn();
|
|
1709
|
+
this.db.exec('COMMIT');
|
|
1710
|
+
return result;
|
|
1711
|
+
} catch (err) {
|
|
1712
|
+
try { this.db.exec('ROLLBACK'); } catch { /* best-effort */ }
|
|
1713
|
+
throw err;
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
// ============================================================
|
|
1718
|
+
// 0.8 Cluster C — materialized-view + atomic-counter primitives
|
|
1719
|
+
// ============================================================
|
|
1720
|
+
|
|
1721
|
+
/**
|
|
1722
|
+
* For each distinct value of the tag whose key prefix is `tagKeyPrefix`
|
|
1723
|
+
* (e.g. `"character="`), return the most-recent active engram. Used by
|
|
1724
|
+
* NovelForge's "latest emotional state per character", "latest motif
|
|
1725
|
+
* phase per motif", "recent chapter summaries".
|
|
1726
|
+
*
|
|
1727
|
+
* `sortBy` defaults to `createdAt`. With `"sequence"`, engrams without a
|
|
1728
|
+
* sequence are excluded (they have no story-time anchor).
|
|
1729
|
+
*/
|
|
1730
|
+
getLatestByTag(opts: {
|
|
1731
|
+
agentId: string;
|
|
1732
|
+
tagKeyPrefix: string; // e.g. "character=", "motif=", "chapter="
|
|
1733
|
+
scopeTagsAll?: string[]; // optional narrowing (e.g. ["topic=motif-use"])
|
|
1734
|
+
retracted?: boolean;
|
|
1735
|
+
sortBy?: 'createdAt' | 'sequence';
|
|
1736
|
+
limit?: number;
|
|
1737
|
+
}): Engram[] {
|
|
1738
|
+
let sql = `SELECT * FROM engrams
|
|
1739
|
+
WHERE agent_id = ?
|
|
1740
|
+
AND retracted = ?
|
|
1741
|
+
AND stage = 'active'
|
|
1742
|
+
AND tags LIKE ?`;
|
|
1743
|
+
const params: any[] = [
|
|
1744
|
+
opts.agentId,
|
|
1745
|
+
opts.retracted ? 1 : 0,
|
|
1746
|
+
`%"${opts.tagKeyPrefix}%`,
|
|
1747
|
+
];
|
|
1748
|
+
if (opts.scopeTagsAll && opts.scopeTagsAll.length > 0) {
|
|
1749
|
+
for (const t of opts.scopeTagsAll) {
|
|
1750
|
+
sql += ' AND tags LIKE ?';
|
|
1751
|
+
params.push(`%"${t}"%`);
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
if (opts.sortBy === 'sequence') {
|
|
1755
|
+
sql += ' AND sequence IS NOT NULL';
|
|
1756
|
+
}
|
|
1757
|
+
sql += ' ORDER BY ' + (opts.sortBy === 'sequence' ? 'sequence DESC, created_at DESC' : 'created_at DESC');
|
|
1758
|
+
|
|
1759
|
+
const rows = this.db.prepare(sql).all(...params) as any[];
|
|
1760
|
+
const engrams = rows.map(r => this.rowToEngram(r));
|
|
1761
|
+
|
|
1762
|
+
// Group by extracted tag value, take first (already sorted newest first).
|
|
1763
|
+
const seen = new Map<string, Engram>();
|
|
1764
|
+
for (const e of engrams) {
|
|
1765
|
+
const value = this.extractTagValue(e.tags, opts.tagKeyPrefix);
|
|
1766
|
+
if (value == null) continue;
|
|
1767
|
+
if (!seen.has(value)) seen.set(value, e);
|
|
1768
|
+
}
|
|
1769
|
+
const out = Array.from(seen.values());
|
|
1770
|
+
return opts.limit ? out.slice(0, opts.limit) : out;
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1773
|
+
/**
|
|
1774
|
+
* Filter engrams by tag-set operators and sort by numeric value extracted
|
|
1775
|
+
* from a tag prefix (e.g. `"weight="` → numeric value of `weight=8`).
|
|
1776
|
+
* Used by NovelForge's "top 40 active promises by weight".
|
|
1777
|
+
*
|
|
1778
|
+
* Missing or unparseable numeric values sort last.
|
|
1779
|
+
*/
|
|
1780
|
+
getTopBy(opts: {
|
|
1781
|
+
agentId: string;
|
|
1782
|
+
sortField: string; // tag prefix, e.g. "weight="
|
|
1783
|
+
order: 'asc' | 'desc';
|
|
1784
|
+
filterTagsAll?: string[];
|
|
1785
|
+
filterTagsAny?: string[];
|
|
1786
|
+
filterTagsNone?: string[];
|
|
1787
|
+
limit?: number;
|
|
1788
|
+
retracted?: boolean;
|
|
1789
|
+
}): Engram[] {
|
|
1790
|
+
// Build the base query inline (mirrors search() but adds the topBy sort).
|
|
1791
|
+
let sql = `SELECT * FROM engrams
|
|
1792
|
+
WHERE agent_id = ?
|
|
1793
|
+
AND retracted = ?
|
|
1794
|
+
AND stage = 'active'
|
|
1795
|
+
AND tags LIKE ?`;
|
|
1796
|
+
const params: any[] = [
|
|
1797
|
+
opts.agentId,
|
|
1798
|
+
opts.retracted ? 1 : 0,
|
|
1799
|
+
`%"${opts.sortField}%`, // must have a tag with this prefix
|
|
1800
|
+
];
|
|
1801
|
+
|
|
1802
|
+
if (opts.filterTagsAll && opts.filterTagsAll.length > 0) {
|
|
1803
|
+
for (const tag of opts.filterTagsAll) {
|
|
1804
|
+
sql += ' AND tags LIKE ?';
|
|
1805
|
+
params.push(`%"${tag}"%`);
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
if (opts.filterTagsAny && opts.filterTagsAny.length > 0) {
|
|
1809
|
+
const ors = opts.filterTagsAny.map(() => 'tags LIKE ?').join(' OR ');
|
|
1810
|
+
sql += ` AND (${ors})`;
|
|
1811
|
+
for (const tag of opts.filterTagsAny) params.push(`%"${tag}"%`);
|
|
1812
|
+
}
|
|
1813
|
+
if (opts.filterTagsNone && opts.filterTagsNone.length > 0) {
|
|
1814
|
+
const ors = opts.filterTagsNone.map(() => 'tags LIKE ?').join(' OR ');
|
|
1815
|
+
sql += ` AND NOT (${ors})`;
|
|
1816
|
+
for (const tag of opts.filterTagsNone) params.push(`%"${tag}"%`);
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
const rows = this.db.prepare(sql).all(...params) as any[];
|
|
1820
|
+
const engrams = rows.map(r => this.rowToEngram(r));
|
|
1821
|
+
|
|
1822
|
+
// Extract numeric value from the sortField tag; missing or NaN sorts last.
|
|
1823
|
+
const valued = engrams.map(e => {
|
|
1824
|
+
const raw = this.extractTagValue(e.tags, opts.sortField);
|
|
1825
|
+
const n = raw == null ? NaN : Number(raw);
|
|
1826
|
+
return { e, n };
|
|
1827
|
+
});
|
|
1828
|
+
valued.sort((a, b) => {
|
|
1829
|
+
const aNaN = Number.isNaN(a.n);
|
|
1830
|
+
const bNaN = Number.isNaN(b.n);
|
|
1831
|
+
if (aNaN && bNaN) return 0;
|
|
1832
|
+
if (aNaN) return 1; // NaN last
|
|
1833
|
+
if (bNaN) return -1;
|
|
1834
|
+
return opts.order === 'asc' ? a.n - b.n : b.n - a.n;
|
|
1835
|
+
});
|
|
1836
|
+
const sorted = valued.map(v => v.e);
|
|
1837
|
+
return opts.limit ? sorted.slice(0, opts.limit) : sorted;
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
/**
|
|
1841
|
+
* Compute the effective state of an engram from referenced events.
|
|
1842
|
+
*
|
|
1843
|
+
* - 'superseded' if engram.supersededBy is set
|
|
1844
|
+
* - else: scan engrams whose references_json names this engram with a
|
|
1845
|
+
* terminal relation type (resolves / subverts / abandons). Take the
|
|
1846
|
+
* latest by createdAt. effectiveState = that type.
|
|
1847
|
+
* - else: 'active'
|
|
1848
|
+
*
|
|
1849
|
+
* Returns `null` if target engram not found.
|
|
1850
|
+
*/
|
|
1851
|
+
resolveEffectiveState(targetEngramId: string): {
|
|
1852
|
+
engram: Engram;
|
|
1853
|
+
effectiveState: 'active' | 'resolved' | 'subverted' | 'abandoned' | 'superseded';
|
|
1854
|
+
resolvingEvents: Array<{ id: string; type: string; createdAt: string }>;
|
|
1855
|
+
} | null {
|
|
1856
|
+
const target = this.getEngram(targetEngramId);
|
|
1857
|
+
if (!target) return null;
|
|
1858
|
+
|
|
1859
|
+
if (target.supersededBy) {
|
|
1860
|
+
return { engram: target, effectiveState: 'superseded', resolvingEvents: [] };
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
// Find candidate referencing engrams. LIKE-filter first to narrow,
|
|
1864
|
+
// then JSON.parse each candidate's references_json to verify the
|
|
1865
|
+
// matchEngramId and type. Cheap enough at NovelForge's scale.
|
|
1866
|
+
const candidates = this.db.prepare(
|
|
1867
|
+
`SELECT id, references_json, created_at FROM engrams
|
|
1868
|
+
WHERE agent_id = ?
|
|
1869
|
+
AND retracted = 0
|
|
1870
|
+
AND stage = 'active'
|
|
1871
|
+
AND references_json IS NOT NULL
|
|
1872
|
+
AND references_json LIKE ?`
|
|
1873
|
+
).all(target.agentId, `%"${targetEngramId}"%`) as Array<{
|
|
1874
|
+
id: string;
|
|
1875
|
+
references_json: string;
|
|
1876
|
+
created_at: string;
|
|
1877
|
+
}>;
|
|
1878
|
+
|
|
1879
|
+
const terminalTypes = new Set(['resolves', 'subverts', 'abandons']);
|
|
1880
|
+
const resolvingEvents: Array<{ id: string; type: string; createdAt: string }> = [];
|
|
1881
|
+
for (const c of candidates) {
|
|
1882
|
+
try {
|
|
1883
|
+
const refs = JSON.parse(c.references_json) as Array<{ type: string; matchEngramId?: string }>;
|
|
1884
|
+
for (const ref of refs) {
|
|
1885
|
+
if (ref.matchEngramId === targetEngramId && terminalTypes.has(ref.type)) {
|
|
1886
|
+
resolvingEvents.push({ id: c.id, type: ref.type, createdAt: c.created_at });
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
|
+
} catch { /* malformed references_json — skip */ }
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
if (resolvingEvents.length === 0) {
|
|
1893
|
+
return { engram: target, effectiveState: 'active', resolvingEvents: [] };
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
// Latest event wins.
|
|
1897
|
+
resolvingEvents.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
1898
|
+
const latestType = resolvingEvents[0]!.type as 'resolves' | 'subverts' | 'abandons';
|
|
1899
|
+
const effectiveState = (
|
|
1900
|
+
latestType === 'resolves' ? 'resolved'
|
|
1901
|
+
: latestType === 'subverts' ? 'subverted'
|
|
1902
|
+
: 'abandoned'
|
|
1903
|
+
);
|
|
1904
|
+
return { engram: target, effectiveState, resolvingEvents };
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
/**
|
|
1908
|
+
* Atomically allocate the next sequence number for an agent.
|
|
1909
|
+
*
|
|
1910
|
+
* Uses BEGIN IMMEDIATE to serialize concurrent callers — better-sqlite3's
|
|
1911
|
+
* transaction() acquires a RESERVED lock which prevents other writers
|
|
1912
|
+
* from racing. Returns MAX(sequence)+1 for the agent, or 1 if no engrams
|
|
1913
|
+
* have a sequence yet.
|
|
1914
|
+
*/
|
|
1915
|
+
allocateNextSequence(agentId: string): number {
|
|
1916
|
+
return this.transaction(() => {
|
|
1917
|
+
const row = this.db.prepare(
|
|
1918
|
+
'SELECT MAX(sequence) AS max_seq FROM engrams WHERE agent_id = ?'
|
|
1919
|
+
).get(agentId) as { max_seq: number | null };
|
|
1920
|
+
return (row?.max_seq ?? 0) + 1;
|
|
1921
|
+
});
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
/**
|
|
1925
|
+
* Extract the value following a tag prefix, e.g. `["weight=8"]` + `"weight="`
|
|
1926
|
+
* → `"8"`. Returns null if no tag with that prefix is present.
|
|
1927
|
+
*/
|
|
1928
|
+
private extractTagValue(tags: string[], tagKeyPrefix: string): string | null {
|
|
1929
|
+
for (const t of tags) {
|
|
1930
|
+
if (t.startsWith(tagKeyPrefix)) {
|
|
1931
|
+
return t.slice(tagKeyPrefix.length);
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
return null;
|
|
1935
|
+
}
|
|
1936
|
+
}
|