@199-bio/engram 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  * Enhanced with temporal decay and salience scoring
5
5
  */
6
6
 
7
- import { EngramDatabase, Memory } from "../storage/database.js";
7
+ import { EngramDatabase, Memory, Digest } from "../storage/database.js";
8
8
  import { KnowledgeGraph } from "../graph/knowledge-graph.js";
9
9
  import { ColBERTRetriever, SimpleRetriever, SearchResult, Document } from "./colbert.js";
10
10
 
@@ -20,8 +20,15 @@ export interface HybridSearchResult {
20
20
  };
21
21
  }
22
22
 
23
+ export interface DigestSearchResult {
24
+ digest: Digest;
25
+ score: number;
26
+ key_memories: Memory[]; // 2-3 source memories that best support this digest
27
+ }
28
+
23
29
  export interface HybridSearchResponse {
24
30
  results: HybridSearchResult[];
31
+ digests: DigestSearchResult[]; // Relevant synthesized context
25
32
  recall_id: string; // For LLM feedback
26
33
  connected_memories: Array<{
27
34
  memory: Memory;
@@ -179,11 +186,15 @@ export class HybridSearch {
179
186
  if (allCandidateIds.size === 0) {
180
187
  return {
181
188
  results: [],
189
+ digests: [],
182
190
  recall_id: recallId,
183
191
  connected_memories: [],
184
192
  };
185
193
  }
186
194
 
195
+ // Search digests via BM25 (top 3 relevant digests)
196
+ const digestResults = this.searchDigests(query, 3);
197
+
187
198
  // Create rankings for RRF
188
199
  const rankings: Map<string, { bm25?: number; semantic?: number; graph?: number; connected?: number }> = new Map();
189
200
 
@@ -327,17 +338,70 @@ export class HybridSearch {
327
338
  // Log this retrieval for deferred learning
328
339
  try {
329
340
  this.db.createRetrievalLog(this.sessionId, recallId, query, allResultIds);
330
- } catch {
331
- // Ignore duplicate recall_id errors (can happen with rapid queries)
341
+ } catch (error) {
342
+ // Only ignore duplicate recall_id errors (UNIQUE constraint), log all others
343
+ const msg = error instanceof Error ? error.message : String(error);
344
+ if (!msg.includes("UNIQUE constraint")) {
345
+ console.error(`[Engram] Failed to create retrieval log: ${msg}`);
346
+ }
347
+ }
348
+
349
+ // TOKEN EFFICIENCY: If digests are returned, reduce memory count
350
+ // Digest provides context (synthesis), memories provide evidence (specifics)
351
+ // Return fewer memories when we have good digest coverage
352
+ let finalResults = results;
353
+ if (digestResults.length > 0) {
354
+ // Get IDs of memories already covered by digests as key_memories
355
+ const coveredByDigests = new Set<string>();
356
+ digestResults.forEach(d => d.key_memories.forEach(m => coveredByDigests.add(m.id)));
357
+
358
+ // Keep memories not already shown as key_memories in digests
359
+ // Also limit to fewer since digests provide the context
360
+ const maxMemoriesWithDigests = Math.max(2, Math.floor(limit / 2));
361
+ finalResults = results
362
+ .filter(r => !coveredByDigests.has(r.memory.id))
363
+ .slice(0, maxMemoriesWithDigests);
332
364
  }
333
365
 
334
366
  return {
335
- results,
367
+ results: finalResults,
368
+ digests: digestResults,
336
369
  recall_id: recallId,
337
370
  connected_memories: connectedMemories,
338
371
  };
339
372
  }
340
373
 
374
+ /**
375
+ * Search digests via BM25 and return with key source memories
376
+ * Returns top N digests with 2-3 representative source memories each
377
+ */
378
+ private searchDigests(query: string, limit: number): DigestSearchResult[] {
379
+ try {
380
+ const digestHits = this.db.searchDigestsBM25(query, limit);
381
+
382
+ return digestHits.map(hit => {
383
+ // Get source memories for this digest, take top 3 most relevant
384
+ const sources = this.db.getDigestSources(hit.id);
385
+ // Sort by importance and recency, take best 3
386
+ const keyMemories = sources
387
+ .sort((a, b) => {
388
+ const scoreA = (a.importance || 0.5) + (a.access_count || 0) * 0.1;
389
+ const scoreB = (b.importance || 0.5) + (b.access_count || 0) * 0.1;
390
+ return scoreB - scoreA;
391
+ })
392
+ .slice(0, 3);
393
+
394
+ return {
395
+ digest: hit,
396
+ score: Math.abs(hit.score), // BM25 returns negative scores
397
+ key_memories: keyMemories,
398
+ };
399
+ });
400
+ } catch {
401
+ return [];
402
+ }
403
+ }
404
+
341
405
  /**
342
406
  * Expanded search when LLM needs more memories
343
407
  * Relaxes constraints and follows weaker connections
@@ -351,7 +415,7 @@ export class HybridSearch {
351
415
  // Get the original retrieval log
352
416
  const log = this.db.getRetrievalLog(recallId);
353
417
  if (!log) {
354
- return { results: [], recall_id: recallId, connected_memories: [] };
418
+ return { results: [], digests: [], recall_id: recallId, connected_memories: [] };
355
419
  }
356
420
 
357
421
  // Search again with relaxed parameters
@@ -119,6 +119,37 @@ export interface RetrievalLog {
119
119
  processed: boolean; // Has this been used for learning?
120
120
  }
121
121
 
122
+ /**
123
+ * Consolidation checkpoint for safe backlog processing
124
+ * Enables resume capability and progress tracking
125
+ */
126
+ export interface ConsolidationCheckpoint {
127
+ id: string;
128
+ run_id: string; // Unique ID for this consolidation run
129
+ phase: "episodes" | "memories" | "decay" | "cleanup" | "complete";
130
+ batches_completed: number;
131
+ batches_total: number;
132
+ memories_processed: number;
133
+ episodes_processed: number;
134
+ digests_created: number;
135
+ contradictions_found: number;
136
+ tokens_used: number;
137
+ estimated_cost_usd: number;
138
+ started_at: Date;
139
+ updated_at: Date;
140
+ completed_at: Date | null;
141
+ error: string | null;
142
+ }
143
+
144
+ /**
145
+ * Consolidation configuration for budget and rate limits
146
+ */
147
+ export interface ConsolidationConfig {
148
+ key: string;
149
+ value: string;
150
+ updated_at: Date;
151
+ }
152
+
122
153
  export class EngramDatabase {
123
154
  private db: Database.Database;
124
155
  private stmtCache: Map<string, Database.Statement> = new Map();
@@ -284,6 +315,34 @@ export class EngramDatabase {
284
315
  CREATE INDEX IF NOT EXISTS idx_digest_sources_memory ON digest_sources(memory_id);
285
316
  `);
286
317
 
318
+ // FTS5 for digest BM25 search
319
+ this.db.exec(`
320
+ CREATE VIRTUAL TABLE IF NOT EXISTS digests_fts USING fts5(
321
+ content,
322
+ topic,
323
+ content='digests',
324
+ content_rowid='rowid'
325
+ );
326
+
327
+ -- Triggers to keep FTS in sync
328
+ CREATE TRIGGER IF NOT EXISTS digests_ai AFTER INSERT ON digests BEGIN
329
+ INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
330
+ END;
331
+
332
+ CREATE TRIGGER IF NOT EXISTS digests_ad AFTER DELETE ON digests BEGIN
333
+ INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
334
+ END;
335
+
336
+ CREATE TRIGGER IF NOT EXISTS digests_au AFTER UPDATE ON digests BEGIN
337
+ INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
338
+ INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
339
+ END;
340
+ `);
341
+
342
+ // Rebuild FTS5 index to sync with content table (necessary for existing databases)
343
+ // This is idempotent and fast for small tables
344
+ this.db.exec(`INSERT INTO digests_fts(digests_fts) VALUES('rebuild');`);
345
+
287
346
  // Contradictions table (detected conflicts)
288
347
  this.db.exec(`
289
348
  CREATE TABLE IF NOT EXISTS contradictions (
@@ -340,6 +399,48 @@ export class EngramDatabase {
340
399
  CREATE INDEX IF NOT EXISTS idx_retrieval_logs_recall ON retrieval_logs(recall_id);
341
400
  CREATE INDEX IF NOT EXISTS idx_retrieval_logs_processed ON retrieval_logs(processed);
342
401
  `);
402
+
403
+ // Consolidation checkpoints for safe backlog processing
404
+ this.db.exec(`
405
+ CREATE TABLE IF NOT EXISTS consolidation_checkpoints (
406
+ id TEXT PRIMARY KEY,
407
+ run_id TEXT NOT NULL UNIQUE,
408
+ phase TEXT NOT NULL CHECK(phase IN ('episodes', 'memories', 'decay', 'cleanup', 'complete')),
409
+ batches_completed INTEGER DEFAULT 0,
410
+ batches_total INTEGER DEFAULT 0,
411
+ memories_processed INTEGER DEFAULT 0,
412
+ episodes_processed INTEGER DEFAULT 0,
413
+ digests_created INTEGER DEFAULT 0,
414
+ contradictions_found INTEGER DEFAULT 0,
415
+ tokens_used INTEGER DEFAULT 0,
416
+ estimated_cost_usd REAL DEFAULT 0,
417
+ started_at DATETIME DEFAULT CURRENT_TIMESTAMP,
418
+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
419
+ completed_at DATETIME,
420
+ error TEXT
421
+ );
422
+
423
+ CREATE INDEX IF NOT EXISTS idx_checkpoints_run ON consolidation_checkpoints(run_id);
424
+ CREATE INDEX IF NOT EXISTS idx_checkpoints_phase ON consolidation_checkpoints(phase);
425
+ `);
426
+
427
+ // Consolidation configuration for budget and rate limits
428
+ this.db.exec(`
429
+ CREATE TABLE IF NOT EXISTS consolidation_config (
430
+ key TEXT PRIMARY KEY,
431
+ value TEXT NOT NULL,
432
+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
433
+ );
434
+
435
+ -- Default configuration values
436
+ INSERT OR IGNORE INTO consolidation_config (key, value) VALUES
437
+ ('daily_budget_usd', '5.0'),
438
+ ('max_batches_per_run', '5'),
439
+ ('delay_between_calls_ms', '2000'),
440
+ ('recovery_mode_threshold', '100'),
441
+ ('error_rate_threshold', '0.3'),
442
+ ('empty_digest_threshold', '0.2');
443
+ `);
343
444
  }
344
445
 
345
446
  /**
@@ -568,6 +669,23 @@ export class EngramDatabase {
568
669
  }));
569
670
  }
570
671
 
672
+ searchDigestsBM25(query: string, limit: number = 10): Array<Digest & { score: number }> {
673
+ const escapedQuery = this.escapeFTS5Query(query);
674
+ const rows = this.stmt(`
675
+ SELECT d.*, bm25(digests_fts) as score
676
+ FROM digests_fts fts
677
+ JOIN digests d ON fts.rowid = d.rowid
678
+ WHERE digests_fts MATCH ?
679
+ ORDER BY score
680
+ LIMIT ?
681
+ `).all(escapedQuery, limit) as Array<Record<string, unknown>>;
682
+
683
+ return rows.map((row) => ({
684
+ ...this.rowToDigest(row),
685
+ score: row.score as number,
686
+ }));
687
+ }
688
+
571
689
  private escapeFTS5Query(query: string): string {
572
690
  // Simple tokenization - split on whitespace, escape special chars
573
691
  const tokens = query
@@ -1183,59 +1301,72 @@ export class EngramDatabase {
1183
1301
 
1184
1302
  /**
1185
1303
  * Record that memories were retrieved together (co-retrieval)
1304
+ * Uses cached statement and transaction for O(n²) efficiency
1186
1305
  */
1187
1306
  recordCoRetrieval(memoryIds: string[]): void {
1188
1307
  if (memoryIds.length < 2) return;
1189
1308
 
1190
- // Update all pairs
1191
- for (let i = 0; i < memoryIds.length; i++) {
1192
- for (let j = i + 1; j < memoryIds.length; j++) {
1193
- const [a, b] = memoryIds[i] < memoryIds[j]
1194
- ? [memoryIds[i], memoryIds[j]]
1195
- : [memoryIds[j], memoryIds[i]];
1196
-
1197
- this.db.prepare(`
1198
- INSERT INTO memory_connections (id, memory_a, memory_b, co_retrievals, last_fired)
1199
- VALUES (?, ?, ?, 1, CURRENT_TIMESTAMP)
1200
- ON CONFLICT(memory_a, memory_b) DO UPDATE SET
1201
- co_retrievals = co_retrievals + 1,
1202
- last_fired = CURRENT_TIMESTAMP
1203
- `).run(randomUUID(), a, b);
1309
+ // Cache statement once (not per iteration)
1310
+ const upsertStmt = this.stmt(`
1311
+ INSERT INTO memory_connections (id, memory_a, memory_b, co_retrievals, last_fired)
1312
+ VALUES (?, ?, ?, 1, CURRENT_TIMESTAMP)
1313
+ ON CONFLICT(memory_a, memory_b) DO UPDATE SET
1314
+ co_retrievals = co_retrievals + 1,
1315
+ last_fired = CURRENT_TIMESTAMP
1316
+ `);
1317
+
1318
+ // Wrap in transaction for batch efficiency
1319
+ this.db.transaction(() => {
1320
+ for (let i = 0; i < memoryIds.length; i++) {
1321
+ for (let j = i + 1; j < memoryIds.length; j++) {
1322
+ const [a, b] = memoryIds[i] < memoryIds[j]
1323
+ ? [memoryIds[i], memoryIds[j]]
1324
+ : [memoryIds[j], memoryIds[i]];
1325
+
1326
+ upsertStmt.run(randomUUID(), a, b);
1327
+ }
1204
1328
  }
1205
- }
1329
+ })();
1206
1330
  }
1207
1331
 
1208
1332
  /**
1209
1333
  * Record that memories were useful together (from LLM feedback)
1210
1334
  * This is what actually strengthens connections
1335
+ * Uses cached statement and transaction for O(n²) efficiency
1211
1336
  */
1212
1337
  recordCoUseful(memoryIds: string[]): void {
1213
1338
  if (memoryIds.length < 2) return;
1214
1339
 
1215
- for (let i = 0; i < memoryIds.length; i++) {
1216
- for (let j = i + 1; j < memoryIds.length; j++) {
1217
- const [a, b] = memoryIds[i] < memoryIds[j]
1218
- ? [memoryIds[i], memoryIds[j]]
1219
- : [memoryIds[j], memoryIds[i]];
1220
-
1221
- // Get current stats to calculate new strength
1222
- const conn = this.getOrCreateConnection(a, b);
1223
- const newCoUseful = conn.co_useful + 1;
1224
- const newCoRetrievals = Math.max(conn.co_retrievals, newCoUseful);
1225
-
1226
- // Calculate strength with diminishing returns
1227
- const usefulRatio = newCoUseful / Math.max(1, newCoRetrievals);
1228
- const diminished = Math.sqrt(usefulRatio);
1229
- const confidence = Math.min(1, newCoUseful / 3); // Require 3+ co-useful for strong connection
1230
- const newStrength = diminished * confidence;
1231
-
1232
- this.db.prepare(`
1233
- UPDATE memory_connections
1234
- SET co_useful = ?, strength = ?, last_fired = CURRENT_TIMESTAMP
1235
- WHERE memory_a = ? AND memory_b = ?
1236
- `).run(newCoUseful, newStrength, a, b);
1340
+ // Cache statement once (not per iteration)
1341
+ const updateStmt = this.stmt(`
1342
+ UPDATE memory_connections
1343
+ SET co_useful = ?, strength = ?, last_fired = CURRENT_TIMESTAMP
1344
+ WHERE memory_a = ? AND memory_b = ?
1345
+ `);
1346
+
1347
+ // Wrap in transaction for batch efficiency
1348
+ this.db.transaction(() => {
1349
+ for (let i = 0; i < memoryIds.length; i++) {
1350
+ for (let j = i + 1; j < memoryIds.length; j++) {
1351
+ const [a, b] = memoryIds[i] < memoryIds[j]
1352
+ ? [memoryIds[i], memoryIds[j]]
1353
+ : [memoryIds[j], memoryIds[i]];
1354
+
1355
+ // Get current stats to calculate new strength
1356
+ const conn = this.getOrCreateConnection(a, b);
1357
+ const newCoUseful = conn.co_useful + 1;
1358
+ const newCoRetrievals = Math.max(conn.co_retrievals, newCoUseful);
1359
+
1360
+ // Calculate strength with diminishing returns
1361
+ const usefulRatio = newCoUseful / Math.max(1, newCoRetrievals);
1362
+ const diminished = Math.sqrt(usefulRatio);
1363
+ const confidence = Math.min(1, newCoUseful / 3); // Require 3+ co-useful for strong connection
1364
+ const newStrength = diminished * confidence;
1365
+
1366
+ updateStmt.run(newCoUseful, newStrength, a, b);
1367
+ }
1237
1368
  }
1238
- }
1369
+ })();
1239
1370
  }
1240
1371
 
1241
1372
  /**
@@ -1328,6 +1459,195 @@ export class EngramDatabase {
1328
1459
  return result.changes;
1329
1460
  }
1330
1461
 
1462
+ // ============ Consolidation Checkpoints ============
1463
+
1464
+ /**
1465
+ * Create a new consolidation checkpoint
1466
+ */
1467
+ createCheckpoint(runId: string, phase: ConsolidationCheckpoint["phase"], batchesTotal: number): ConsolidationCheckpoint {
1468
+ const id = randomUUID();
1469
+ this.db.prepare(`
1470
+ INSERT INTO consolidation_checkpoints (id, run_id, phase, batches_total)
1471
+ VALUES (?, ?, ?, ?)
1472
+ `).run(id, runId, phase, batchesTotal);
1473
+ return this.getCheckpoint(runId)!;
1474
+ }
1475
+
1476
+ /**
1477
+ * Get checkpoint by run_id
1478
+ */
1479
+ getCheckpoint(runId: string): ConsolidationCheckpoint | null {
1480
+ const row = this.stmt("SELECT * FROM consolidation_checkpoints WHERE run_id = ?").get(runId) as Record<string, unknown> | undefined;
1481
+ return row ? this.rowToCheckpoint(row) : null;
1482
+ }
1483
+
1484
+ /**
1485
+ * Get the latest incomplete checkpoint (for resume)
1486
+ */
1487
+ getIncompleteCheckpoint(): ConsolidationCheckpoint | null {
1488
+ const row = this.stmt(`
1489
+ SELECT * FROM consolidation_checkpoints
1490
+ WHERE completed_at IS NULL AND error IS NULL
1491
+ ORDER BY started_at DESC LIMIT 1
1492
+ `).get() as Record<string, unknown> | undefined;
1493
+ return row ? this.rowToCheckpoint(row) : null;
1494
+ }
1495
+
1496
+ /**
1497
+ * Update checkpoint progress
1498
+ */
1499
+ updateCheckpoint(
1500
+ runId: string,
1501
+ updates: Partial<Pick<ConsolidationCheckpoint,
1502
+ "phase" | "batches_completed" | "batches_total" | "memories_processed" |
1503
+ "episodes_processed" | "digests_created" | "contradictions_found" |
1504
+ "tokens_used" | "estimated_cost_usd" | "error"
1505
+ >>
1506
+ ): ConsolidationCheckpoint | null {
1507
+ const sets: string[] = ["updated_at = CURRENT_TIMESTAMP"];
1508
+ const values: unknown[] = [];
1509
+
1510
+ if (updates.phase !== undefined) {
1511
+ sets.push("phase = ?");
1512
+ values.push(updates.phase);
1513
+ }
1514
+ if (updates.batches_completed !== undefined) {
1515
+ sets.push("batches_completed = ?");
1516
+ values.push(updates.batches_completed);
1517
+ }
1518
+ if (updates.batches_total !== undefined) {
1519
+ sets.push("batches_total = ?");
1520
+ values.push(updates.batches_total);
1521
+ }
1522
+ if (updates.memories_processed !== undefined) {
1523
+ sets.push("memories_processed = ?");
1524
+ values.push(updates.memories_processed);
1525
+ }
1526
+ if (updates.episodes_processed !== undefined) {
1527
+ sets.push("episodes_processed = ?");
1528
+ values.push(updates.episodes_processed);
1529
+ }
1530
+ if (updates.digests_created !== undefined) {
1531
+ sets.push("digests_created = ?");
1532
+ values.push(updates.digests_created);
1533
+ }
1534
+ if (updates.contradictions_found !== undefined) {
1535
+ sets.push("contradictions_found = ?");
1536
+ values.push(updates.contradictions_found);
1537
+ }
1538
+ if (updates.tokens_used !== undefined) {
1539
+ sets.push("tokens_used = ?");
1540
+ values.push(updates.tokens_used);
1541
+ }
1542
+ if (updates.estimated_cost_usd !== undefined) {
1543
+ sets.push("estimated_cost_usd = ?");
1544
+ values.push(updates.estimated_cost_usd);
1545
+ }
1546
+ if (updates.error !== undefined) {
1547
+ sets.push("error = ?");
1548
+ values.push(updates.error);
1549
+ }
1550
+
1551
+ values.push(runId);
1552
+ this.db.prepare(`UPDATE consolidation_checkpoints SET ${sets.join(", ")} WHERE run_id = ?`).run(...values);
1553
+ return this.getCheckpoint(runId);
1554
+ }
1555
+
1556
+ /**
1557
+ * Mark checkpoint as complete
1558
+ */
1559
+ completeCheckpoint(runId: string): void {
1560
+ this.db.prepare(`
1561
+ UPDATE consolidation_checkpoints
1562
+ SET phase = 'complete', completed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
1563
+ WHERE run_id = ?
1564
+ `).run(runId);
1565
+ }
1566
+
1567
+ /**
1568
+ * Get recent checkpoints for reporting
1569
+ */
1570
+ getRecentCheckpoints(limit: number = 10): ConsolidationCheckpoint[] {
1571
+ const rows = this.stmt(`
1572
+ SELECT * FROM consolidation_checkpoints
1573
+ ORDER BY started_at DESC LIMIT ?
1574
+ `).all(limit) as Record<string, unknown>[];
1575
+ return rows.map(r => this.rowToCheckpoint(r));
1576
+ }
1577
+
1578
+ /**
1579
+ * Get total spending in the last 24 hours
1580
+ */
1581
+ getDailySpending(): number {
1582
+ const row = this.stmt(`
1583
+ SELECT COALESCE(SUM(estimated_cost_usd), 0) as total
1584
+ FROM consolidation_checkpoints
1585
+ WHERE started_at >= datetime('now', '-1 day')
1586
+ `).get() as { total: number };
1587
+ return row.total;
1588
+ }
1589
+
1590
+ // ============ Consolidation Config ============
1591
+
1592
+ /**
1593
+ * Get a config value
1594
+ */
1595
+ getConfig(key: string): string | null {
1596
+ const row = this.stmt("SELECT value FROM consolidation_config WHERE key = ?").get(key) as { value: string } | undefined;
1597
+ return row?.value ?? null;
1598
+ }
1599
+
1600
+ /**
1601
+ * Get a config value as number
1602
+ */
1603
+ getConfigNumber(key: string, defaultValue: number): number {
1604
+ const value = this.getConfig(key);
1605
+ return value ? parseFloat(value) : defaultValue;
1606
+ }
1607
+
1608
+ /**
1609
+ * Set a config value
1610
+ */
1611
+ setConfig(key: string, value: string): void {
1612
+ this.db.prepare(`
1613
+ INSERT INTO consolidation_config (key, value, updated_at)
1614
+ VALUES (?, ?, CURRENT_TIMESTAMP)
1615
+ ON CONFLICT(key) DO UPDATE SET value = ?, updated_at = CURRENT_TIMESTAMP
1616
+ `).run(key, value, value);
1617
+ }
1618
+
1619
+ /**
1620
+ * Get all config values
1621
+ */
1622
+ getAllConfig(): Record<string, string> {
1623
+ const rows = this.stmt("SELECT key, value FROM consolidation_config").all() as Array<{ key: string; value: string }>;
1624
+ const config: Record<string, string> = {};
1625
+ for (const row of rows) {
1626
+ config[row.key] = row.value;
1627
+ }
1628
+ return config;
1629
+ }
1630
+
1631
+ private rowToCheckpoint(row: Record<string, unknown>): ConsolidationCheckpoint {
1632
+ return {
1633
+ id: row.id as string,
1634
+ run_id: row.run_id as string,
1635
+ phase: row.phase as ConsolidationCheckpoint["phase"],
1636
+ batches_completed: row.batches_completed as number,
1637
+ batches_total: row.batches_total as number,
1638
+ memories_processed: row.memories_processed as number,
1639
+ episodes_processed: row.episodes_processed as number,
1640
+ digests_created: row.digests_created as number,
1641
+ contradictions_found: row.contradictions_found as number,
1642
+ tokens_used: row.tokens_used as number,
1643
+ estimated_cost_usd: row.estimated_cost_usd as number,
1644
+ started_at: new Date(row.started_at as string),
1645
+ updated_at: new Date(row.updated_at as string),
1646
+ completed_at: row.completed_at ? new Date(row.completed_at as string) : null,
1647
+ error: row.error as string | null,
1648
+ };
1649
+ }
1650
+
1331
1651
  // ============ Statistics ============
1332
1652
 
1333
1653
  getStats(): {
@@ -0,0 +1,111 @@
1
+ /**
2
+ * HTTP Transport for Engram MCP Server
3
+ * Uses StreamableHTTPServerTransport in stateless mode for Railway deployment
4
+ */
5
+
6
+ import http from "http";
7
+ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
8
+ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
9
+
10
+ interface HttpServerOptions {
11
+ port: number;
12
+ server: Server;
13
+ }
14
+
15
+ /**
16
+ * Start HTTP server with MCP transport
17
+ * Returns a promise that resolves when server is listening
18
+ */
19
+ export async function startHttpServer(options: HttpServerOptions): Promise<http.Server> {
20
+ const { port, server } = options;
21
+
22
+ // Create stateless transport (sessionIdGenerator: undefined)
23
+ // Stateless mode is perfect for Railway - no session management needed
24
+ const transport = new StreamableHTTPServerTransport({
25
+ sessionIdGenerator: undefined,
26
+ });
27
+
28
+ // Connect MCP server to transport
29
+ await server.connect(transport);
30
+
31
+ // Create HTTP server
32
+ const httpServer = http.createServer(async (req, res) => {
33
+ // CORS headers for remote clients (ElevenLabs, etc.)
34
+ res.setHeader("Access-Control-Allow-Origin", "*");
35
+ res.setHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS");
36
+ res.setHeader("Access-Control-Allow-Headers", "Content-Type, mcp-session-id");
37
+
38
+ if (req.method === "OPTIONS") {
39
+ res.writeHead(204);
40
+ res.end();
41
+ return;
42
+ }
43
+
44
+ const url = new URL(req.url || "/", `http://localhost:${port}`);
45
+
46
+ // Health check endpoint (Railway uses this)
47
+ if (url.pathname === "/health" && req.method === "GET") {
48
+ res.writeHead(200, { "Content-Type": "application/json" });
49
+ res.end(JSON.stringify({
50
+ status: "ok",
51
+ transport: "http",
52
+ version: "0.10.0"
53
+ }));
54
+ return;
55
+ }
56
+
57
+ // Root endpoint - service info
58
+ if (url.pathname === "/" && req.method === "GET") {
59
+ res.writeHead(200, { "Content-Type": "application/json" });
60
+ res.end(JSON.stringify({
61
+ name: "engram",
62
+ description: "MCP memory server with hybrid search",
63
+ version: "0.10.0",
64
+ transport: "streamable-http",
65
+ endpoints: {
66
+ mcp: "/mcp",
67
+ health: "/health",
68
+ },
69
+ }));
70
+ return;
71
+ }
72
+
73
+ // MCP endpoint - handles both POST (messages) and GET (SSE stream)
74
+ if (url.pathname === "/mcp") {
75
+ try {
76
+ await transport.handleRequest(req, res);
77
+ } catch (error) {
78
+ console.error("[Engram HTTP] Error handling MCP request:", error);
79
+ if (!res.headersSent) {
80
+ res.writeHead(500, { "Content-Type": "application/json" });
81
+ res.end(JSON.stringify({ error: "Internal server error" }));
82
+ }
83
+ }
84
+ return;
85
+ }
86
+
87
+ // SSE alias (for clients expecting /sse endpoint)
88
+ if (url.pathname === "/sse") {
89
+ // Redirect to /mcp which handles SSE via GET
90
+ res.writeHead(307, { "Location": "/mcp" });
91
+ res.end();
92
+ return;
93
+ }
94
+
95
+ // 404 for unknown paths
96
+ res.writeHead(404, { "Content-Type": "application/json" });
97
+ res.end(JSON.stringify({ error: "Not found" }));
98
+ });
99
+
100
+ // Start listening
101
+ return new Promise((resolve, reject) => {
102
+ httpServer.once("error", reject);
103
+ httpServer.listen(port, () => {
104
+ console.error(`[Engram] MCP HTTP server running on port ${port}`);
105
+ console.error(`[Engram] Endpoints:`);
106
+ console.error(`[Engram] POST http://localhost:${port}/mcp - MCP protocol`);
107
+ console.error(`[Engram] GET http://localhost:${port}/health - Health check`);
108
+ resolve(httpServer);
109
+ });
110
+ });
111
+ }