@199-bio/engram 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +5 -0
- package/boba-prompt.md +107 -0
- package/dist/consolidation/consolidator.d.ts.map +1 -1
- package/dist/consolidation/plan.d.ts.map +1 -0
- package/dist/index.js +170 -9
- package/dist/retrieval/hybrid.d.ts.map +1 -1
- package/dist/storage/database.d.ts.map +1 -1
- package/dist/transport/http.d.ts.map +1 -0
- package/dist/transport/index.d.ts.map +1 -0
- package/dist/web/chat-handler.d.ts.map +1 -1
- package/nixpacks.toml +11 -0
- package/package.json +2 -1
- package/railway.json +13 -0
- package/src/consolidation/consolidator.ts +381 -29
- package/src/consolidation/plan.ts +444 -0
- package/src/index.ts +181 -10
- package/src/retrieval/hybrid.ts +69 -5
- package/src/storage/database.ts +358 -38
- package/src/transport/http.ts +111 -0
- package/src/transport/index.ts +24 -0
- package/src/web/chat-handler.ts +116 -70
- package/src/web/static/app.js +612 -360
- package/src/web/static/index.html +377 -130
- package/src/web/static/style.css +1249 -672
package/src/retrieval/hybrid.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Enhanced with temporal decay and salience scoring
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { EngramDatabase, Memory } from "../storage/database.js";
|
|
7
|
+
import { EngramDatabase, Memory, Digest } from "../storage/database.js";
|
|
8
8
|
import { KnowledgeGraph } from "../graph/knowledge-graph.js";
|
|
9
9
|
import { ColBERTRetriever, SimpleRetriever, SearchResult, Document } from "./colbert.js";
|
|
10
10
|
|
|
@@ -20,8 +20,15 @@ export interface HybridSearchResult {
|
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
export interface DigestSearchResult {
|
|
24
|
+
digest: Digest;
|
|
25
|
+
score: number;
|
|
26
|
+
key_memories: Memory[]; // 2-3 source memories that best support this digest
|
|
27
|
+
}
|
|
28
|
+
|
|
23
29
|
export interface HybridSearchResponse {
|
|
24
30
|
results: HybridSearchResult[];
|
|
31
|
+
digests: DigestSearchResult[]; // Relevant synthesized context
|
|
25
32
|
recall_id: string; // For LLM feedback
|
|
26
33
|
connected_memories: Array<{
|
|
27
34
|
memory: Memory;
|
|
@@ -179,11 +186,15 @@ export class HybridSearch {
|
|
|
179
186
|
if (allCandidateIds.size === 0) {
|
|
180
187
|
return {
|
|
181
188
|
results: [],
|
|
189
|
+
digests: [],
|
|
182
190
|
recall_id: recallId,
|
|
183
191
|
connected_memories: [],
|
|
184
192
|
};
|
|
185
193
|
}
|
|
186
194
|
|
|
195
|
+
// Search digests via BM25 (top 3 relevant digests)
|
|
196
|
+
const digestResults = this.searchDigests(query, 3);
|
|
197
|
+
|
|
187
198
|
// Create rankings for RRF
|
|
188
199
|
const rankings: Map<string, { bm25?: number; semantic?: number; graph?: number; connected?: number }> = new Map();
|
|
189
200
|
|
|
@@ -327,17 +338,70 @@ export class HybridSearch {
|
|
|
327
338
|
// Log this retrieval for deferred learning
|
|
328
339
|
try {
|
|
329
340
|
this.db.createRetrievalLog(this.sessionId, recallId, query, allResultIds);
|
|
330
|
-
} catch {
|
|
331
|
-
//
|
|
341
|
+
} catch (error) {
|
|
342
|
+
// Only ignore duplicate recall_id errors (UNIQUE constraint), log all others
|
|
343
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
344
|
+
if (!msg.includes("UNIQUE constraint")) {
|
|
345
|
+
console.error(`[Engram] Failed to create retrieval log: ${msg}`);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// TOKEN EFFICIENCY: If digests are returned, reduce memory count
|
|
350
|
+
// Digest provides context (synthesis), memories provide evidence (specifics)
|
|
351
|
+
// Return fewer memories when we have good digest coverage
|
|
352
|
+
let finalResults = results;
|
|
353
|
+
if (digestResults.length > 0) {
|
|
354
|
+
// Get IDs of memories already covered by digests as key_memories
|
|
355
|
+
const coveredByDigests = new Set<string>();
|
|
356
|
+
digestResults.forEach(d => d.key_memories.forEach(m => coveredByDigests.add(m.id)));
|
|
357
|
+
|
|
358
|
+
// Keep memories not already shown as key_memories in digests
|
|
359
|
+
// Also limit to fewer since digests provide the context
|
|
360
|
+
const maxMemoriesWithDigests = Math.max(2, Math.floor(limit / 2));
|
|
361
|
+
finalResults = results
|
|
362
|
+
.filter(r => !coveredByDigests.has(r.memory.id))
|
|
363
|
+
.slice(0, maxMemoriesWithDigests);
|
|
332
364
|
}
|
|
333
365
|
|
|
334
366
|
return {
|
|
335
|
-
results,
|
|
367
|
+
results: finalResults,
|
|
368
|
+
digests: digestResults,
|
|
336
369
|
recall_id: recallId,
|
|
337
370
|
connected_memories: connectedMemories,
|
|
338
371
|
};
|
|
339
372
|
}
|
|
340
373
|
|
|
374
|
+
/**
|
|
375
|
+
* Search digests via BM25 and return with key source memories
|
|
376
|
+
* Returns top N digests with 2-3 representative source memories each
|
|
377
|
+
*/
|
|
378
|
+
private searchDigests(query: string, limit: number): DigestSearchResult[] {
|
|
379
|
+
try {
|
|
380
|
+
const digestHits = this.db.searchDigestsBM25(query, limit);
|
|
381
|
+
|
|
382
|
+
return digestHits.map(hit => {
|
|
383
|
+
// Get source memories for this digest, take top 3 most relevant
|
|
384
|
+
const sources = this.db.getDigestSources(hit.id);
|
|
385
|
+
// Sort by importance and recency, take best 3
|
|
386
|
+
const keyMemories = sources
|
|
387
|
+
.sort((a, b) => {
|
|
388
|
+
const scoreA = (a.importance || 0.5) + (a.access_count || 0) * 0.1;
|
|
389
|
+
const scoreB = (b.importance || 0.5) + (b.access_count || 0) * 0.1;
|
|
390
|
+
return scoreB - scoreA;
|
|
391
|
+
})
|
|
392
|
+
.slice(0, 3);
|
|
393
|
+
|
|
394
|
+
return {
|
|
395
|
+
digest: hit,
|
|
396
|
+
score: Math.abs(hit.score), // BM25 returns negative scores
|
|
397
|
+
key_memories: keyMemories,
|
|
398
|
+
};
|
|
399
|
+
});
|
|
400
|
+
} catch {
|
|
401
|
+
return [];
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
341
405
|
/**
|
|
342
406
|
* Expanded search when LLM needs more memories
|
|
343
407
|
* Relaxes constraints and follows weaker connections
|
|
@@ -351,7 +415,7 @@ export class HybridSearch {
|
|
|
351
415
|
// Get the original retrieval log
|
|
352
416
|
const log = this.db.getRetrievalLog(recallId);
|
|
353
417
|
if (!log) {
|
|
354
|
-
return { results: [], recall_id: recallId, connected_memories: [] };
|
|
418
|
+
return { results: [], digests: [], recall_id: recallId, connected_memories: [] };
|
|
355
419
|
}
|
|
356
420
|
|
|
357
421
|
// Search again with relaxed parameters
|
package/src/storage/database.ts
CHANGED
|
@@ -119,6 +119,37 @@ export interface RetrievalLog {
|
|
|
119
119
|
processed: boolean; // Has this been used for learning?
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Consolidation checkpoint for safe backlog processing
|
|
124
|
+
* Enables resume capability and progress tracking
|
|
125
|
+
*/
|
|
126
|
+
export interface ConsolidationCheckpoint {
|
|
127
|
+
id: string;
|
|
128
|
+
run_id: string; // Unique ID for this consolidation run
|
|
129
|
+
phase: "episodes" | "memories" | "decay" | "cleanup" | "complete";
|
|
130
|
+
batches_completed: number;
|
|
131
|
+
batches_total: number;
|
|
132
|
+
memories_processed: number;
|
|
133
|
+
episodes_processed: number;
|
|
134
|
+
digests_created: number;
|
|
135
|
+
contradictions_found: number;
|
|
136
|
+
tokens_used: number;
|
|
137
|
+
estimated_cost_usd: number;
|
|
138
|
+
started_at: Date;
|
|
139
|
+
updated_at: Date;
|
|
140
|
+
completed_at: Date | null;
|
|
141
|
+
error: string | null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Consolidation configuration for budget and rate limits
|
|
146
|
+
*/
|
|
147
|
+
export interface ConsolidationConfig {
|
|
148
|
+
key: string;
|
|
149
|
+
value: string;
|
|
150
|
+
updated_at: Date;
|
|
151
|
+
}
|
|
152
|
+
|
|
122
153
|
export class EngramDatabase {
|
|
123
154
|
private db: Database.Database;
|
|
124
155
|
private stmtCache: Map<string, Database.Statement> = new Map();
|
|
@@ -284,6 +315,34 @@ export class EngramDatabase {
|
|
|
284
315
|
CREATE INDEX IF NOT EXISTS idx_digest_sources_memory ON digest_sources(memory_id);
|
|
285
316
|
`);
|
|
286
317
|
|
|
318
|
+
// FTS5 for digest BM25 search
|
|
319
|
+
this.db.exec(`
|
|
320
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS digests_fts USING fts5(
|
|
321
|
+
content,
|
|
322
|
+
topic,
|
|
323
|
+
content='digests',
|
|
324
|
+
content_rowid='rowid'
|
|
325
|
+
);
|
|
326
|
+
|
|
327
|
+
-- Triggers to keep FTS in sync
|
|
328
|
+
CREATE TRIGGER IF NOT EXISTS digests_ai AFTER INSERT ON digests BEGIN
|
|
329
|
+
INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
|
|
330
|
+
END;
|
|
331
|
+
|
|
332
|
+
CREATE TRIGGER IF NOT EXISTS digests_ad AFTER DELETE ON digests BEGIN
|
|
333
|
+
INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
|
|
334
|
+
END;
|
|
335
|
+
|
|
336
|
+
CREATE TRIGGER IF NOT EXISTS digests_au AFTER UPDATE ON digests BEGIN
|
|
337
|
+
INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
|
|
338
|
+
INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
|
|
339
|
+
END;
|
|
340
|
+
`);
|
|
341
|
+
|
|
342
|
+
// Rebuild FTS5 index to sync with content table (necessary for existing databases)
|
|
343
|
+
// This is idempotent and fast for small tables
|
|
344
|
+
this.db.exec(`INSERT INTO digests_fts(digests_fts) VALUES('rebuild');`);
|
|
345
|
+
|
|
287
346
|
// Contradictions table (detected conflicts)
|
|
288
347
|
this.db.exec(`
|
|
289
348
|
CREATE TABLE IF NOT EXISTS contradictions (
|
|
@@ -340,6 +399,48 @@ export class EngramDatabase {
|
|
|
340
399
|
CREATE INDEX IF NOT EXISTS idx_retrieval_logs_recall ON retrieval_logs(recall_id);
|
|
341
400
|
CREATE INDEX IF NOT EXISTS idx_retrieval_logs_processed ON retrieval_logs(processed);
|
|
342
401
|
`);
|
|
402
|
+
|
|
403
|
+
// Consolidation checkpoints for safe backlog processing
|
|
404
|
+
this.db.exec(`
|
|
405
|
+
CREATE TABLE IF NOT EXISTS consolidation_checkpoints (
|
|
406
|
+
id TEXT PRIMARY KEY,
|
|
407
|
+
run_id TEXT NOT NULL UNIQUE,
|
|
408
|
+
phase TEXT NOT NULL CHECK(phase IN ('episodes', 'memories', 'decay', 'cleanup', 'complete')),
|
|
409
|
+
batches_completed INTEGER DEFAULT 0,
|
|
410
|
+
batches_total INTEGER DEFAULT 0,
|
|
411
|
+
memories_processed INTEGER DEFAULT 0,
|
|
412
|
+
episodes_processed INTEGER DEFAULT 0,
|
|
413
|
+
digests_created INTEGER DEFAULT 0,
|
|
414
|
+
contradictions_found INTEGER DEFAULT 0,
|
|
415
|
+
tokens_used INTEGER DEFAULT 0,
|
|
416
|
+
estimated_cost_usd REAL DEFAULT 0,
|
|
417
|
+
started_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
418
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
419
|
+
completed_at DATETIME,
|
|
420
|
+
error TEXT
|
|
421
|
+
);
|
|
422
|
+
|
|
423
|
+
CREATE INDEX IF NOT EXISTS idx_checkpoints_run ON consolidation_checkpoints(run_id);
|
|
424
|
+
CREATE INDEX IF NOT EXISTS idx_checkpoints_phase ON consolidation_checkpoints(phase);
|
|
425
|
+
`);
|
|
426
|
+
|
|
427
|
+
// Consolidation configuration for budget and rate limits
|
|
428
|
+
this.db.exec(`
|
|
429
|
+
CREATE TABLE IF NOT EXISTS consolidation_config (
|
|
430
|
+
key TEXT PRIMARY KEY,
|
|
431
|
+
value TEXT NOT NULL,
|
|
432
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
433
|
+
);
|
|
434
|
+
|
|
435
|
+
-- Default configuration values
|
|
436
|
+
INSERT OR IGNORE INTO consolidation_config (key, value) VALUES
|
|
437
|
+
('daily_budget_usd', '5.0'),
|
|
438
|
+
('max_batches_per_run', '5'),
|
|
439
|
+
('delay_between_calls_ms', '2000'),
|
|
440
|
+
('recovery_mode_threshold', '100'),
|
|
441
|
+
('error_rate_threshold', '0.3'),
|
|
442
|
+
('empty_digest_threshold', '0.2');
|
|
443
|
+
`);
|
|
343
444
|
}
|
|
344
445
|
|
|
345
446
|
/**
|
|
@@ -568,6 +669,23 @@ export class EngramDatabase {
|
|
|
568
669
|
}));
|
|
569
670
|
}
|
|
570
671
|
|
|
672
|
+
searchDigestsBM25(query: string, limit: number = 10): Array<Digest & { score: number }> {
|
|
673
|
+
const escapedQuery = this.escapeFTS5Query(query);
|
|
674
|
+
const rows = this.stmt(`
|
|
675
|
+
SELECT d.*, bm25(digests_fts) as score
|
|
676
|
+
FROM digests_fts fts
|
|
677
|
+
JOIN digests d ON fts.rowid = d.rowid
|
|
678
|
+
WHERE digests_fts MATCH ?
|
|
679
|
+
ORDER BY score
|
|
680
|
+
LIMIT ?
|
|
681
|
+
`).all(escapedQuery, limit) as Array<Record<string, unknown>>;
|
|
682
|
+
|
|
683
|
+
return rows.map((row) => ({
|
|
684
|
+
...this.rowToDigest(row),
|
|
685
|
+
score: row.score as number,
|
|
686
|
+
}));
|
|
687
|
+
}
|
|
688
|
+
|
|
571
689
|
private escapeFTS5Query(query: string): string {
|
|
572
690
|
// Simple tokenization - split on whitespace, escape special chars
|
|
573
691
|
const tokens = query
|
|
@@ -1183,59 +1301,72 @@ export class EngramDatabase {
|
|
|
1183
1301
|
|
|
1184
1302
|
/**
|
|
1185
1303
|
* Record that memories were retrieved together (co-retrieval)
|
|
1304
|
+
* Uses cached statement and transaction for O(n²) efficiency
|
|
1186
1305
|
*/
|
|
1187
1306
|
recordCoRetrieval(memoryIds: string[]): void {
|
|
1188
1307
|
if (memoryIds.length < 2) return;
|
|
1189
1308
|
|
|
1190
|
-
//
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1309
|
+
// Cache statement once (not per iteration)
|
|
1310
|
+
const upsertStmt = this.stmt(`
|
|
1311
|
+
INSERT INTO memory_connections (id, memory_a, memory_b, co_retrievals, last_fired)
|
|
1312
|
+
VALUES (?, ?, ?, 1, CURRENT_TIMESTAMP)
|
|
1313
|
+
ON CONFLICT(memory_a, memory_b) DO UPDATE SET
|
|
1314
|
+
co_retrievals = co_retrievals + 1,
|
|
1315
|
+
last_fired = CURRENT_TIMESTAMP
|
|
1316
|
+
`);
|
|
1317
|
+
|
|
1318
|
+
// Wrap in transaction for batch efficiency
|
|
1319
|
+
this.db.transaction(() => {
|
|
1320
|
+
for (let i = 0; i < memoryIds.length; i++) {
|
|
1321
|
+
for (let j = i + 1; j < memoryIds.length; j++) {
|
|
1322
|
+
const [a, b] = memoryIds[i] < memoryIds[j]
|
|
1323
|
+
? [memoryIds[i], memoryIds[j]]
|
|
1324
|
+
: [memoryIds[j], memoryIds[i]];
|
|
1325
|
+
|
|
1326
|
+
upsertStmt.run(randomUUID(), a, b);
|
|
1327
|
+
}
|
|
1204
1328
|
}
|
|
1205
|
-
}
|
|
1329
|
+
})();
|
|
1206
1330
|
}
|
|
1207
1331
|
|
|
1208
1332
|
/**
|
|
1209
1333
|
* Record that memories were useful together (from LLM feedback)
|
|
1210
1334
|
* This is what actually strengthens connections
|
|
1335
|
+
* Uses cached statement and transaction for O(n²) efficiency
|
|
1211
1336
|
*/
|
|
1212
1337
|
recordCoUseful(memoryIds: string[]): void {
|
|
1213
1338
|
if (memoryIds.length < 2) return;
|
|
1214
1339
|
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1340
|
+
// Cache statement once (not per iteration)
|
|
1341
|
+
const updateStmt = this.stmt(`
|
|
1342
|
+
UPDATE memory_connections
|
|
1343
|
+
SET co_useful = ?, strength = ?, last_fired = CURRENT_TIMESTAMP
|
|
1344
|
+
WHERE memory_a = ? AND memory_b = ?
|
|
1345
|
+
`);
|
|
1346
|
+
|
|
1347
|
+
// Wrap in transaction for batch efficiency
|
|
1348
|
+
this.db.transaction(() => {
|
|
1349
|
+
for (let i = 0; i < memoryIds.length; i++) {
|
|
1350
|
+
for (let j = i + 1; j < memoryIds.length; j++) {
|
|
1351
|
+
const [a, b] = memoryIds[i] < memoryIds[j]
|
|
1352
|
+
? [memoryIds[i], memoryIds[j]]
|
|
1353
|
+
: [memoryIds[j], memoryIds[i]];
|
|
1354
|
+
|
|
1355
|
+
// Get current stats to calculate new strength
|
|
1356
|
+
const conn = this.getOrCreateConnection(a, b);
|
|
1357
|
+
const newCoUseful = conn.co_useful + 1;
|
|
1358
|
+
const newCoRetrievals = Math.max(conn.co_retrievals, newCoUseful);
|
|
1359
|
+
|
|
1360
|
+
// Calculate strength with diminishing returns
|
|
1361
|
+
const usefulRatio = newCoUseful / Math.max(1, newCoRetrievals);
|
|
1362
|
+
const diminished = Math.sqrt(usefulRatio);
|
|
1363
|
+
const confidence = Math.min(1, newCoUseful / 3); // Require 3+ co-useful for strong connection
|
|
1364
|
+
const newStrength = diminished * confidence;
|
|
1365
|
+
|
|
1366
|
+
updateStmt.run(newCoUseful, newStrength, a, b);
|
|
1367
|
+
}
|
|
1237
1368
|
}
|
|
1238
|
-
}
|
|
1369
|
+
})();
|
|
1239
1370
|
}
|
|
1240
1371
|
|
|
1241
1372
|
/**
|
|
@@ -1328,6 +1459,195 @@ export class EngramDatabase {
|
|
|
1328
1459
|
return result.changes;
|
|
1329
1460
|
}
|
|
1330
1461
|
|
|
1462
|
+
// ============ Consolidation Checkpoints ============
|
|
1463
|
+
|
|
1464
|
+
/**
|
|
1465
|
+
* Create a new consolidation checkpoint
|
|
1466
|
+
*/
|
|
1467
|
+
createCheckpoint(runId: string, phase: ConsolidationCheckpoint["phase"], batchesTotal: number): ConsolidationCheckpoint {
|
|
1468
|
+
const id = randomUUID();
|
|
1469
|
+
this.db.prepare(`
|
|
1470
|
+
INSERT INTO consolidation_checkpoints (id, run_id, phase, batches_total)
|
|
1471
|
+
VALUES (?, ?, ?, ?)
|
|
1472
|
+
`).run(id, runId, phase, batchesTotal);
|
|
1473
|
+
return this.getCheckpoint(runId)!;
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
/**
|
|
1477
|
+
* Get checkpoint by run_id
|
|
1478
|
+
*/
|
|
1479
|
+
getCheckpoint(runId: string): ConsolidationCheckpoint | null {
|
|
1480
|
+
const row = this.stmt("SELECT * FROM consolidation_checkpoints WHERE run_id = ?").get(runId) as Record<string, unknown> | undefined;
|
|
1481
|
+
return row ? this.rowToCheckpoint(row) : null;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
/**
|
|
1485
|
+
* Get the latest incomplete checkpoint (for resume)
|
|
1486
|
+
*/
|
|
1487
|
+
getIncompleteCheckpoint(): ConsolidationCheckpoint | null {
|
|
1488
|
+
const row = this.stmt(`
|
|
1489
|
+
SELECT * FROM consolidation_checkpoints
|
|
1490
|
+
WHERE completed_at IS NULL AND error IS NULL
|
|
1491
|
+
ORDER BY started_at DESC LIMIT 1
|
|
1492
|
+
`).get() as Record<string, unknown> | undefined;
|
|
1493
|
+
return row ? this.rowToCheckpoint(row) : null;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
/**
|
|
1497
|
+
* Update checkpoint progress
|
|
1498
|
+
*/
|
|
1499
|
+
updateCheckpoint(
|
|
1500
|
+
runId: string,
|
|
1501
|
+
updates: Partial<Pick<ConsolidationCheckpoint,
|
|
1502
|
+
"phase" | "batches_completed" | "batches_total" | "memories_processed" |
|
|
1503
|
+
"episodes_processed" | "digests_created" | "contradictions_found" |
|
|
1504
|
+
"tokens_used" | "estimated_cost_usd" | "error"
|
|
1505
|
+
>>
|
|
1506
|
+
): ConsolidationCheckpoint | null {
|
|
1507
|
+
const sets: string[] = ["updated_at = CURRENT_TIMESTAMP"];
|
|
1508
|
+
const values: unknown[] = [];
|
|
1509
|
+
|
|
1510
|
+
if (updates.phase !== undefined) {
|
|
1511
|
+
sets.push("phase = ?");
|
|
1512
|
+
values.push(updates.phase);
|
|
1513
|
+
}
|
|
1514
|
+
if (updates.batches_completed !== undefined) {
|
|
1515
|
+
sets.push("batches_completed = ?");
|
|
1516
|
+
values.push(updates.batches_completed);
|
|
1517
|
+
}
|
|
1518
|
+
if (updates.batches_total !== undefined) {
|
|
1519
|
+
sets.push("batches_total = ?");
|
|
1520
|
+
values.push(updates.batches_total);
|
|
1521
|
+
}
|
|
1522
|
+
if (updates.memories_processed !== undefined) {
|
|
1523
|
+
sets.push("memories_processed = ?");
|
|
1524
|
+
values.push(updates.memories_processed);
|
|
1525
|
+
}
|
|
1526
|
+
if (updates.episodes_processed !== undefined) {
|
|
1527
|
+
sets.push("episodes_processed = ?");
|
|
1528
|
+
values.push(updates.episodes_processed);
|
|
1529
|
+
}
|
|
1530
|
+
if (updates.digests_created !== undefined) {
|
|
1531
|
+
sets.push("digests_created = ?");
|
|
1532
|
+
values.push(updates.digests_created);
|
|
1533
|
+
}
|
|
1534
|
+
if (updates.contradictions_found !== undefined) {
|
|
1535
|
+
sets.push("contradictions_found = ?");
|
|
1536
|
+
values.push(updates.contradictions_found);
|
|
1537
|
+
}
|
|
1538
|
+
if (updates.tokens_used !== undefined) {
|
|
1539
|
+
sets.push("tokens_used = ?");
|
|
1540
|
+
values.push(updates.tokens_used);
|
|
1541
|
+
}
|
|
1542
|
+
if (updates.estimated_cost_usd !== undefined) {
|
|
1543
|
+
sets.push("estimated_cost_usd = ?");
|
|
1544
|
+
values.push(updates.estimated_cost_usd);
|
|
1545
|
+
}
|
|
1546
|
+
if (updates.error !== undefined) {
|
|
1547
|
+
sets.push("error = ?");
|
|
1548
|
+
values.push(updates.error);
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
values.push(runId);
|
|
1552
|
+
this.db.prepare(`UPDATE consolidation_checkpoints SET ${sets.join(", ")} WHERE run_id = ?`).run(...values);
|
|
1553
|
+
return this.getCheckpoint(runId);
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/**
|
|
1557
|
+
* Mark checkpoint as complete
|
|
1558
|
+
*/
|
|
1559
|
+
completeCheckpoint(runId: string): void {
|
|
1560
|
+
this.db.prepare(`
|
|
1561
|
+
UPDATE consolidation_checkpoints
|
|
1562
|
+
SET phase = 'complete', completed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
1563
|
+
WHERE run_id = ?
|
|
1564
|
+
`).run(runId);
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
/**
|
|
1568
|
+
* Get recent checkpoints for reporting
|
|
1569
|
+
*/
|
|
1570
|
+
getRecentCheckpoints(limit: number = 10): ConsolidationCheckpoint[] {
|
|
1571
|
+
const rows = this.stmt(`
|
|
1572
|
+
SELECT * FROM consolidation_checkpoints
|
|
1573
|
+
ORDER BY started_at DESC LIMIT ?
|
|
1574
|
+
`).all(limit) as Record<string, unknown>[];
|
|
1575
|
+
return rows.map(r => this.rowToCheckpoint(r));
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
/**
|
|
1579
|
+
* Get total spending in the last 24 hours
|
|
1580
|
+
*/
|
|
1581
|
+
getDailySpending(): number {
|
|
1582
|
+
const row = this.stmt(`
|
|
1583
|
+
SELECT COALESCE(SUM(estimated_cost_usd), 0) as total
|
|
1584
|
+
FROM consolidation_checkpoints
|
|
1585
|
+
WHERE started_at >= datetime('now', '-1 day')
|
|
1586
|
+
`).get() as { total: number };
|
|
1587
|
+
return row.total;
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
// ============ Consolidation Config ============
|
|
1591
|
+
|
|
1592
|
+
/**
|
|
1593
|
+
* Get a config value
|
|
1594
|
+
*/
|
|
1595
|
+
getConfig(key: string): string | null {
|
|
1596
|
+
const row = this.stmt("SELECT value FROM consolidation_config WHERE key = ?").get(key) as { value: string } | undefined;
|
|
1597
|
+
return row?.value ?? null;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Get a config value as number
|
|
1602
|
+
*/
|
|
1603
|
+
getConfigNumber(key: string, defaultValue: number): number {
|
|
1604
|
+
const value = this.getConfig(key);
|
|
1605
|
+
return value ? parseFloat(value) : defaultValue;
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
/**
|
|
1609
|
+
* Set a config value
|
|
1610
|
+
*/
|
|
1611
|
+
setConfig(key: string, value: string): void {
|
|
1612
|
+
this.db.prepare(`
|
|
1613
|
+
INSERT INTO consolidation_config (key, value, updated_at)
|
|
1614
|
+
VALUES (?, ?, CURRENT_TIMESTAMP)
|
|
1615
|
+
ON CONFLICT(key) DO UPDATE SET value = ?, updated_at = CURRENT_TIMESTAMP
|
|
1616
|
+
`).run(key, value, value);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
/**
|
|
1620
|
+
* Get all config values
|
|
1621
|
+
*/
|
|
1622
|
+
getAllConfig(): Record<string, string> {
|
|
1623
|
+
const rows = this.stmt("SELECT key, value FROM consolidation_config").all() as Array<{ key: string; value: string }>;
|
|
1624
|
+
const config: Record<string, string> = {};
|
|
1625
|
+
for (const row of rows) {
|
|
1626
|
+
config[row.key] = row.value;
|
|
1627
|
+
}
|
|
1628
|
+
return config;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
private rowToCheckpoint(row: Record<string, unknown>): ConsolidationCheckpoint {
|
|
1632
|
+
return {
|
|
1633
|
+
id: row.id as string,
|
|
1634
|
+
run_id: row.run_id as string,
|
|
1635
|
+
phase: row.phase as ConsolidationCheckpoint["phase"],
|
|
1636
|
+
batches_completed: row.batches_completed as number,
|
|
1637
|
+
batches_total: row.batches_total as number,
|
|
1638
|
+
memories_processed: row.memories_processed as number,
|
|
1639
|
+
episodes_processed: row.episodes_processed as number,
|
|
1640
|
+
digests_created: row.digests_created as number,
|
|
1641
|
+
contradictions_found: row.contradictions_found as number,
|
|
1642
|
+
tokens_used: row.tokens_used as number,
|
|
1643
|
+
estimated_cost_usd: row.estimated_cost_usd as number,
|
|
1644
|
+
started_at: new Date(row.started_at as string),
|
|
1645
|
+
updated_at: new Date(row.updated_at as string),
|
|
1646
|
+
completed_at: row.completed_at ? new Date(row.completed_at as string) : null,
|
|
1647
|
+
error: row.error as string | null,
|
|
1648
|
+
};
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1331
1651
|
// ============ Statistics ============
|
|
1332
1652
|
|
|
1333
1653
|
getStats(): {
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP Transport for Engram MCP Server
|
|
3
|
+
* Uses StreamableHTTPServerTransport in stateless mode for Railway deployment
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import http from "http";
|
|
7
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
8
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
9
|
+
|
|
10
|
+
interface HttpServerOptions {
|
|
11
|
+
port: number;
|
|
12
|
+
server: Server;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Start HTTP server with MCP transport
|
|
17
|
+
* Returns a promise that resolves when server is listening
|
|
18
|
+
*/
|
|
19
|
+
export async function startHttpServer(options: HttpServerOptions): Promise<http.Server> {
|
|
20
|
+
const { port, server } = options;
|
|
21
|
+
|
|
22
|
+
// Create stateless transport (sessionIdGenerator: undefined)
|
|
23
|
+
// Stateless mode is perfect for Railway - no session management needed
|
|
24
|
+
const transport = new StreamableHTTPServerTransport({
|
|
25
|
+
sessionIdGenerator: undefined,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
// Connect MCP server to transport
|
|
29
|
+
await server.connect(transport);
|
|
30
|
+
|
|
31
|
+
// Create HTTP server
|
|
32
|
+
const httpServer = http.createServer(async (req, res) => {
|
|
33
|
+
// CORS headers for remote clients (ElevenLabs, etc.)
|
|
34
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
35
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS");
|
|
36
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type, mcp-session-id");
|
|
37
|
+
|
|
38
|
+
if (req.method === "OPTIONS") {
|
|
39
|
+
res.writeHead(204);
|
|
40
|
+
res.end();
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const url = new URL(req.url || "/", `http://localhost:${port}`);
|
|
45
|
+
|
|
46
|
+
// Health check endpoint (Railway uses this)
|
|
47
|
+
if (url.pathname === "/health" && req.method === "GET") {
|
|
48
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
49
|
+
res.end(JSON.stringify({
|
|
50
|
+
status: "ok",
|
|
51
|
+
transport: "http",
|
|
52
|
+
version: "0.10.0"
|
|
53
|
+
}));
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Root endpoint - service info
|
|
58
|
+
if (url.pathname === "/" && req.method === "GET") {
|
|
59
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
60
|
+
res.end(JSON.stringify({
|
|
61
|
+
name: "engram",
|
|
62
|
+
description: "MCP memory server with hybrid search",
|
|
63
|
+
version: "0.10.0",
|
|
64
|
+
transport: "streamable-http",
|
|
65
|
+
endpoints: {
|
|
66
|
+
mcp: "/mcp",
|
|
67
|
+
health: "/health",
|
|
68
|
+
},
|
|
69
|
+
}));
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// MCP endpoint - handles both POST (messages) and GET (SSE stream)
|
|
74
|
+
if (url.pathname === "/mcp") {
|
|
75
|
+
try {
|
|
76
|
+
await transport.handleRequest(req, res);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
console.error("[Engram HTTP] Error handling MCP request:", error);
|
|
79
|
+
if (!res.headersSent) {
|
|
80
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
81
|
+
res.end(JSON.stringify({ error: "Internal server error" }));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// SSE alias (for clients expecting /sse endpoint)
|
|
88
|
+
if (url.pathname === "/sse") {
|
|
89
|
+
// Redirect to /mcp which handles SSE via GET
|
|
90
|
+
res.writeHead(307, { "Location": "/mcp" });
|
|
91
|
+
res.end();
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 404 for unknown paths
|
|
96
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
97
|
+
res.end(JSON.stringify({ error: "Not found" }));
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Start listening
|
|
101
|
+
return new Promise((resolve, reject) => {
|
|
102
|
+
httpServer.once("error", reject);
|
|
103
|
+
httpServer.listen(port, () => {
|
|
104
|
+
console.error(`[Engram] MCP HTTP server running on port ${port}`);
|
|
105
|
+
console.error(`[Engram] Endpoints:`);
|
|
106
|
+
console.error(`[Engram] POST http://localhost:${port}/mcp - MCP protocol`);
|
|
107
|
+
console.error(`[Engram] GET http://localhost:${port}/health - Health check`);
|
|
108
|
+
resolve(httpServer);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|