engrm 0.4.27 → 0.4.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -227,7 +227,7 @@ The MCP server exposes tools that supported agents can call directly:
227
227
  | `load_handoff` | Open a saved handoff as a resume point for a new session |
228
228
  | `refresh_chat_recall` | Rehydrate the separate chat lane from a Claude transcript when a long session feels under-captured |
229
229
  | `recent_chat` | Inspect the separate synced chat lane without mixing it into durable memory |
230
- | `search_chat` | Search recent chat recall separately from reusable memory observations |
230
+ | `search_chat` | Search recent chat recall with hybrid lexical + semantic matching, separately from reusable memory observations |
231
231
  | `search_recall` | Search durable memory and chat recall together when you do not want to guess the right lane |
232
232
  | `plugin_catalog` | Inspect Engrm plugin manifests for memory-aware integrations |
233
233
  | `save_plugin_memory` | Save reduced plugin output with stable Engrm provenance |
@@ -362,6 +362,8 @@ What each tool is good for:
362
362
  - `memory_console` gives the quickest project snapshot, including whether continuity is `fresh`, `thin`, or `cold`
363
363
  - `memory_console`, `project_memory_index`, and `session_context` now also show whether project chat recall is transcript-backed or only hook-captured
364
364
  - when chat continuity is only hook-captured, the workbench and startup hints now prefer `refresh_chat_recall`
365
+ - the workbench and startup hints now also prefer `search_recall` as the first “what were we just talking about?” path when recent prompts/chat/observations exist
366
+ - `search_chat` now uses hybrid lexical + semantic ranking when sqlite-vec and local embeddings are available, so recent conversation recall is less dependent on exact wording
365
367
  - `activity_feed` shows the merged chronology across prompts, tools, chat, handoffs, observations, and summaries
366
368
  - `recent_sessions` helps you pick a session worth opening
367
369
  - `session_story` reconstructs one session in detail, including handoffs and chat recall
@@ -369,7 +371,7 @@ What each tool is good for:
369
371
  - `session_tool_memory` shows which tool calls in one session turned into reusable memory and which did not
370
372
  - `project_memory_index` shows typed memory by repo, including continuity state and hot files
371
373
  - `workspace_memory_index` shows coverage across all repos on the machine
372
- - `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, so weak OpenClaw recall is easier to diagnose and refresh
374
+ - `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, and `search_chat` will also mark when semantic ranking was available, so weak OpenClaw recall is easier to diagnose and refresh
373
375
 
374
376
  ### Thin Tool Workflow
375
377
 
package/dist/cli.js CHANGED
@@ -708,6 +708,17 @@ var MIGRATIONS = [
708
708
  ON chat_messages(session_id, transcript_index)
709
709
  WHERE transcript_index IS NOT NULL;
710
710
  `
711
+ },
712
+ {
713
+ version: 18,
714
+ description: "Add sqlite-vec semantic search for chat recall",
715
+ condition: (db) => isVecExtensionLoaded(db),
716
+ sql: `
717
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
718
+ chat_message_id INTEGER PRIMARY KEY,
719
+ embedding FLOAT[384]
720
+ );
721
+ `
711
722
  }
712
723
  ];
713
724
  function isVecExtensionLoaded(db) {
@@ -781,6 +792,9 @@ function inferLegacySchemaVersion(db) {
781
792
  if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
782
793
  version = Math.max(version, 17);
783
794
  }
795
+ if (tableExists(db, "vec_chat_messages")) {
796
+ version = Math.max(version, 18);
797
+ }
784
798
  return version;
785
799
  }
786
800
  function runMigrations(db) {
@@ -897,6 +911,20 @@ function ensureChatMessageColumns(db) {
897
911
  db.exec("PRAGMA user_version = 17");
898
912
  }
899
913
  }
914
+ function ensureChatVectorTable(db) {
915
+ if (!isVecExtensionLoaded(db))
916
+ return;
917
+ db.exec(`
918
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
919
+ chat_message_id INTEGER PRIMARY KEY,
920
+ embedding FLOAT[384]
921
+ );
922
+ `);
923
+ const current = getSchemaVersion(db);
924
+ if (current < 18) {
925
+ db.exec("PRAGMA user_version = 18");
926
+ }
927
+ }
900
928
  function ensureSyncOutboxSupportsChatMessages(db) {
901
929
  if (syncOutboxSupportsChatMessages(db)) {
902
930
  const current = getSchemaVersion(db);
@@ -1110,6 +1138,7 @@ class MemDatabase {
1110
1138
  ensureObservationTypes(this.db);
1111
1139
  ensureSessionSummaryColumns(this.db);
1112
1140
  ensureChatMessageColumns(this.db);
1141
+ ensureChatVectorTable(this.db);
1113
1142
  ensureSyncOutboxSupportsChatMessages(this.db);
1114
1143
  }
1115
1144
  loadVecExtension() {
@@ -1476,6 +1505,14 @@ class MemDatabase {
1476
1505
  getChatMessageByRemoteSourceId(remoteSourceId) {
1477
1506
  return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
1478
1507
  }
1508
+ getChatMessagesByIds(ids) {
1509
+ if (ids.length === 0)
1510
+ return [];
1511
+ const placeholders = ids.map(() => "?").join(",");
1512
+ const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
1513
+ const order = new Map(ids.map((id, index) => [id, index]));
1514
+ return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
1515
+ }
1479
1516
  getSessionChatMessages(sessionId, limit = 50) {
1480
1517
  return this.db.query(`SELECT * FROM chat_messages
1481
1518
  WHERE session_id = ?
@@ -1552,6 +1589,39 @@ class MemDatabase {
1552
1589
  ORDER BY created_at_epoch DESC, id DESC
1553
1590
  LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
1554
1591
  }
1592
+ vecChatInsert(chatMessageId, embedding) {
1593
+ if (!this.vecAvailable)
1594
+ return;
1595
+ this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
1596
+ }
1597
+ searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
1598
+ if (!this.vecAvailable)
1599
+ return [];
1600
+ const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
1601
+ const visibilityClause = userId ? " AND c.user_id = ?" : "";
1602
+ const transcriptPreference = `
1603
+ AND (
1604
+ c.source_kind = 'transcript'
1605
+ OR NOT EXISTS (
1606
+ SELECT 1 FROM chat_messages t2
1607
+ WHERE t2.session_id = c.session_id
1608
+ AND t2.source_kind = 'transcript'
1609
+ )
1610
+ )`;
1611
+ if (projectId !== null) {
1612
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1613
+ FROM vec_chat_messages v
1614
+ JOIN chat_messages c ON c.id = v.chat_message_id
1615
+ WHERE v.embedding MATCH ?
1616
+ AND k = ?
1617
+ AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
1618
+ }
1619
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1620
+ FROM vec_chat_messages v
1621
+ JOIN chat_messages c ON c.id = v.chat_message_id
1622
+ WHERE v.embedding MATCH ?
1623
+ AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
1624
+ }
1555
1625
  getTranscriptChatMessage(sessionId, transcriptIndex) {
1556
1626
  return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
1557
1627
  }
@@ -2296,6 +2366,17 @@ var MIGRATIONS2 = [
2296
2366
  ON chat_messages(session_id, transcript_index)
2297
2367
  WHERE transcript_index IS NOT NULL;
2298
2368
  `
2369
+ },
2370
+ {
2371
+ version: 18,
2372
+ description: "Add sqlite-vec semantic search for chat recall",
2373
+ condition: (db) => isVecExtensionLoaded2(db),
2374
+ sql: `
2375
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
2376
+ chat_message_id INTEGER PRIMARY KEY,
2377
+ embedding FLOAT[384]
2378
+ );
2379
+ `
2299
2380
  }
2300
2381
  ];
2301
2382
  function isVecExtensionLoaded2(db) {
@@ -3288,6 +3369,9 @@ function composeEmbeddingText(obs) {
3288
3369
 
3289
3370
  `);
3290
3371
  }
3372
+ function composeChatEmbeddingText(text) {
3373
+ return text.replace(/\s+/g, " ").trim().slice(0, 2000);
3374
+ }
3291
3375
  async function getPipeline() {
3292
3376
  if (_pipeline)
3293
3377
  return _pipeline;
@@ -510,6 +510,9 @@ function composeEmbeddingText(obs) {
510
510
 
511
511
  `);
512
512
  }
513
+ function composeChatEmbeddingText(text) {
514
+ return text.replace(/\s+/g, " ").trim().slice(0, 2000);
515
+ }
513
516
  async function getPipeline() {
514
517
  if (_pipeline)
515
518
  return _pipeline;
@@ -1541,6 +1544,17 @@ var MIGRATIONS = [
1541
1544
  ON chat_messages(session_id, transcript_index)
1542
1545
  WHERE transcript_index IS NOT NULL;
1543
1546
  `
1547
+ },
1548
+ {
1549
+ version: 18,
1550
+ description: "Add sqlite-vec semantic search for chat recall",
1551
+ condition: (db) => isVecExtensionLoaded(db),
1552
+ sql: `
1553
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
1554
+ chat_message_id INTEGER PRIMARY KEY,
1555
+ embedding FLOAT[384]
1556
+ );
1557
+ `
1544
1558
  }
1545
1559
  ];
1546
1560
  function isVecExtensionLoaded(db) {
@@ -1614,6 +1628,9 @@ function inferLegacySchemaVersion(db) {
1614
1628
  if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
1615
1629
  version = Math.max(version, 17);
1616
1630
  }
1631
+ if (tableExists(db, "vec_chat_messages")) {
1632
+ version = Math.max(version, 18);
1633
+ }
1617
1634
  return version;
1618
1635
  }
1619
1636
  function runMigrations(db) {
@@ -1730,6 +1747,20 @@ function ensureChatMessageColumns(db) {
1730
1747
  db.exec("PRAGMA user_version = 17");
1731
1748
  }
1732
1749
  }
1750
+ function ensureChatVectorTable(db) {
1751
+ if (!isVecExtensionLoaded(db))
1752
+ return;
1753
+ db.exec(`
1754
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
1755
+ chat_message_id INTEGER PRIMARY KEY,
1756
+ embedding FLOAT[384]
1757
+ );
1758
+ `);
1759
+ const current = getSchemaVersion(db);
1760
+ if (current < 18) {
1761
+ db.exec("PRAGMA user_version = 18");
1762
+ }
1763
+ }
1733
1764
  function ensureSyncOutboxSupportsChatMessages(db) {
1734
1765
  if (syncOutboxSupportsChatMessages(db)) {
1735
1766
  const current = getSchemaVersion(db);
@@ -1943,6 +1974,7 @@ class MemDatabase {
1943
1974
  ensureObservationTypes(this.db);
1944
1975
  ensureSessionSummaryColumns(this.db);
1945
1976
  ensureChatMessageColumns(this.db);
1977
+ ensureChatVectorTable(this.db);
1946
1978
  ensureSyncOutboxSupportsChatMessages(this.db);
1947
1979
  }
1948
1980
  loadVecExtension() {
@@ -2309,6 +2341,14 @@ class MemDatabase {
2309
2341
  getChatMessageByRemoteSourceId(remoteSourceId) {
2310
2342
  return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
2311
2343
  }
2344
+ getChatMessagesByIds(ids) {
2345
+ if (ids.length === 0)
2346
+ return [];
2347
+ const placeholders = ids.map(() => "?").join(",");
2348
+ const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
2349
+ const order = new Map(ids.map((id, index) => [id, index]));
2350
+ return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
2351
+ }
2312
2352
  getSessionChatMessages(sessionId, limit = 50) {
2313
2353
  return this.db.query(`SELECT * FROM chat_messages
2314
2354
  WHERE session_id = ?
@@ -2385,6 +2425,39 @@ class MemDatabase {
2385
2425
  ORDER BY created_at_epoch DESC, id DESC
2386
2426
  LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
2387
2427
  }
2428
+ vecChatInsert(chatMessageId, embedding) {
2429
+ if (!this.vecAvailable)
2430
+ return;
2431
+ this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
2432
+ }
2433
+ searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
2434
+ if (!this.vecAvailable)
2435
+ return [];
2436
+ const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
2437
+ const visibilityClause = userId ? " AND c.user_id = ?" : "";
2438
+ const transcriptPreference = `
2439
+ AND (
2440
+ c.source_kind = 'transcript'
2441
+ OR NOT EXISTS (
2442
+ SELECT 1 FROM chat_messages t2
2443
+ WHERE t2.session_id = c.session_id
2444
+ AND t2.source_kind = 'transcript'
2445
+ )
2446
+ )`;
2447
+ if (projectId !== null) {
2448
+ return this.db.query(`SELECT v.chat_message_id, v.distance
2449
+ FROM vec_chat_messages v
2450
+ JOIN chat_messages c ON c.id = v.chat_message_id
2451
+ WHERE v.embedding MATCH ?
2452
+ AND k = ?
2453
+ AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
2454
+ }
2455
+ return this.db.query(`SELECT v.chat_message_id, v.distance
2456
+ FROM vec_chat_messages v
2457
+ JOIN chat_messages c ON c.id = v.chat_message_id
2458
+ WHERE v.embedding MATCH ?
2459
+ AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
2460
+ }
2388
2461
  getTranscriptChatMessage(sessionId, transcriptIndex) {
2389
2462
  return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
2390
2463
  }
@@ -886,6 +886,17 @@ var MIGRATIONS = [
886
886
  ON chat_messages(session_id, transcript_index)
887
887
  WHERE transcript_index IS NOT NULL;
888
888
  `
889
+ },
890
+ {
891
+ version: 18,
892
+ description: "Add sqlite-vec semantic search for chat recall",
893
+ condition: (db) => isVecExtensionLoaded(db),
894
+ sql: `
895
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
896
+ chat_message_id INTEGER PRIMARY KEY,
897
+ embedding FLOAT[384]
898
+ );
899
+ `
889
900
  }
890
901
  ];
891
902
  function isVecExtensionLoaded(db) {
@@ -959,6 +970,9 @@ function inferLegacySchemaVersion(db) {
959
970
  if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
960
971
  version = Math.max(version, 17);
961
972
  }
973
+ if (tableExists(db, "vec_chat_messages")) {
974
+ version = Math.max(version, 18);
975
+ }
962
976
  return version;
963
977
  }
964
978
  function runMigrations(db) {
@@ -1075,6 +1089,20 @@ function ensureChatMessageColumns(db) {
1075
1089
  db.exec("PRAGMA user_version = 17");
1076
1090
  }
1077
1091
  }
1092
+ function ensureChatVectorTable(db) {
1093
+ if (!isVecExtensionLoaded(db))
1094
+ return;
1095
+ db.exec(`
1096
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
1097
+ chat_message_id INTEGER PRIMARY KEY,
1098
+ embedding FLOAT[384]
1099
+ );
1100
+ `);
1101
+ const current = getSchemaVersion(db);
1102
+ if (current < 18) {
1103
+ db.exec("PRAGMA user_version = 18");
1104
+ }
1105
+ }
1078
1106
  function ensureSyncOutboxSupportsChatMessages(db) {
1079
1107
  if (syncOutboxSupportsChatMessages(db)) {
1080
1108
  const current = getSchemaVersion(db);
@@ -1288,6 +1316,7 @@ class MemDatabase {
1288
1316
  ensureObservationTypes(this.db);
1289
1317
  ensureSessionSummaryColumns(this.db);
1290
1318
  ensureChatMessageColumns(this.db);
1319
+ ensureChatVectorTable(this.db);
1291
1320
  ensureSyncOutboxSupportsChatMessages(this.db);
1292
1321
  }
1293
1322
  loadVecExtension() {
@@ -1654,6 +1683,14 @@ class MemDatabase {
1654
1683
  getChatMessageByRemoteSourceId(remoteSourceId) {
1655
1684
  return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
1656
1685
  }
1686
+ getChatMessagesByIds(ids) {
1687
+ if (ids.length === 0)
1688
+ return [];
1689
+ const placeholders = ids.map(() => "?").join(",");
1690
+ const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
1691
+ const order = new Map(ids.map((id, index) => [id, index]));
1692
+ return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
1693
+ }
1657
1694
  getSessionChatMessages(sessionId, limit = 50) {
1658
1695
  return this.db.query(`SELECT * FROM chat_messages
1659
1696
  WHERE session_id = ?
@@ -1730,6 +1767,39 @@ class MemDatabase {
1730
1767
  ORDER BY created_at_epoch DESC, id DESC
1731
1768
  LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
1732
1769
  }
1770
+ vecChatInsert(chatMessageId, embedding) {
1771
+ if (!this.vecAvailable)
1772
+ return;
1773
+ this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
1774
+ }
1775
+ searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
1776
+ if (!this.vecAvailable)
1777
+ return [];
1778
+ const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
1779
+ const visibilityClause = userId ? " AND c.user_id = ?" : "";
1780
+ const transcriptPreference = `
1781
+ AND (
1782
+ c.source_kind = 'transcript'
1783
+ OR NOT EXISTS (
1784
+ SELECT 1 FROM chat_messages t2
1785
+ WHERE t2.session_id = c.session_id
1786
+ AND t2.source_kind = 'transcript'
1787
+ )
1788
+ )`;
1789
+ if (projectId !== null) {
1790
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1791
+ FROM vec_chat_messages v
1792
+ JOIN chat_messages c ON c.id = v.chat_message_id
1793
+ WHERE v.embedding MATCH ?
1794
+ AND k = ?
1795
+ AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
1796
+ }
1797
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1798
+ FROM vec_chat_messages v
1799
+ JOIN chat_messages c ON c.id = v.chat_message_id
1800
+ WHERE v.embedding MATCH ?
1801
+ AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
1802
+ }
1733
1803
  getTranscriptChatMessage(sessionId, transcriptIndex) {
1734
1804
  return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
1735
1805
  }
@@ -2514,6 +2584,9 @@ function composeEmbeddingText(obs) {
2514
2584
 
2515
2585
  `);
2516
2586
  }
2587
+ function composeChatEmbeddingText(text) {
2588
+ return text.replace(/\s+/g, " ").trim().slice(0, 2000);
2589
+ }
2517
2590
  async function getPipeline() {
2518
2591
  if (_pipeline)
2519
2592
  return _pipeline;
@@ -3701,7 +3774,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
3701
3774
  }
3702
3775
  return messages;
3703
3776
  }
3704
- function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
3777
+ async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
3705
3778
  const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
3706
3779
  ...message,
3707
3780
  text: message.text.trim()
@@ -3731,6 +3804,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
3731
3804
  transcript_index: transcriptIndex
3732
3805
  });
3733
3806
  db.addToOutbox("chat_message", row.id);
3807
+ if (db.vecAvailable) {
3808
+ const embedding = await embedText(composeChatEmbeddingText(message.text));
3809
+ if (embedding) {
3810
+ db.vecChatInsert(row.id, embedding);
3811
+ }
3812
+ }
3734
3813
  imported++;
3735
3814
  }
3736
3815
  return { imported, total: messages.length };
@@ -4256,7 +4335,7 @@ async function main() {
4256
4335
  try {
4257
4336
  if (event.session_id) {
4258
4337
  persistRawToolChronology(event, config.user_id, config.device_id);
4259
- syncTranscriptChat(db, config, event.session_id, event.cwd);
4338
+ await syncTranscriptChat(db, config, event.session_id, event.cwd);
4260
4339
  }
4261
4340
  const textToScan = extractScanText(event);
4262
4341
  if (textToScan) {
@@ -680,6 +680,17 @@ var MIGRATIONS = [
680
680
  ON chat_messages(session_id, transcript_index)
681
681
  WHERE transcript_index IS NOT NULL;
682
682
  `
683
+ },
684
+ {
685
+ version: 18,
686
+ description: "Add sqlite-vec semantic search for chat recall",
687
+ condition: (db) => isVecExtensionLoaded(db),
688
+ sql: `
689
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
690
+ chat_message_id INTEGER PRIMARY KEY,
691
+ embedding FLOAT[384]
692
+ );
693
+ `
683
694
  }
684
695
  ];
685
696
  function isVecExtensionLoaded(db) {
@@ -753,6 +764,9 @@ function inferLegacySchemaVersion(db) {
753
764
  if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
754
765
  version = Math.max(version, 17);
755
766
  }
767
+ if (tableExists(db, "vec_chat_messages")) {
768
+ version = Math.max(version, 18);
769
+ }
756
770
  return version;
757
771
  }
758
772
  function runMigrations(db) {
@@ -869,6 +883,20 @@ function ensureChatMessageColumns(db) {
869
883
  db.exec("PRAGMA user_version = 17");
870
884
  }
871
885
  }
886
+ function ensureChatVectorTable(db) {
887
+ if (!isVecExtensionLoaded(db))
888
+ return;
889
+ db.exec(`
890
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
891
+ chat_message_id INTEGER PRIMARY KEY,
892
+ embedding FLOAT[384]
893
+ );
894
+ `);
895
+ const current = getSchemaVersion(db);
896
+ if (current < 18) {
897
+ db.exec("PRAGMA user_version = 18");
898
+ }
899
+ }
872
900
  function ensureSyncOutboxSupportsChatMessages(db) {
873
901
  if (syncOutboxSupportsChatMessages(db)) {
874
902
  const current = getSchemaVersion(db);
@@ -1082,6 +1110,7 @@ class MemDatabase {
1082
1110
  ensureObservationTypes(this.db);
1083
1111
  ensureSessionSummaryColumns(this.db);
1084
1112
  ensureChatMessageColumns(this.db);
1113
+ ensureChatVectorTable(this.db);
1085
1114
  ensureSyncOutboxSupportsChatMessages(this.db);
1086
1115
  }
1087
1116
  loadVecExtension() {
@@ -1448,6 +1477,14 @@ class MemDatabase {
1448
1477
  getChatMessageByRemoteSourceId(remoteSourceId) {
1449
1478
  return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
1450
1479
  }
1480
+ getChatMessagesByIds(ids) {
1481
+ if (ids.length === 0)
1482
+ return [];
1483
+ const placeholders = ids.map(() => "?").join(",");
1484
+ const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
1485
+ const order = new Map(ids.map((id, index) => [id, index]));
1486
+ return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
1487
+ }
1451
1488
  getSessionChatMessages(sessionId, limit = 50) {
1452
1489
  return this.db.query(`SELECT * FROM chat_messages
1453
1490
  WHERE session_id = ?
@@ -1524,6 +1561,39 @@ class MemDatabase {
1524
1561
  ORDER BY created_at_epoch DESC, id DESC
1525
1562
  LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
1526
1563
  }
1564
+ vecChatInsert(chatMessageId, embedding) {
1565
+ if (!this.vecAvailable)
1566
+ return;
1567
+ this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
1568
+ }
1569
+ searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
1570
+ if (!this.vecAvailable)
1571
+ return [];
1572
+ const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
1573
+ const visibilityClause = userId ? " AND c.user_id = ?" : "";
1574
+ const transcriptPreference = `
1575
+ AND (
1576
+ c.source_kind = 'transcript'
1577
+ OR NOT EXISTS (
1578
+ SELECT 1 FROM chat_messages t2
1579
+ WHERE t2.session_id = c.session_id
1580
+ AND t2.source_kind = 'transcript'
1581
+ )
1582
+ )`;
1583
+ if (projectId !== null) {
1584
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1585
+ FROM vec_chat_messages v
1586
+ JOIN chat_messages c ON c.id = v.chat_message_id
1587
+ WHERE v.embedding MATCH ?
1588
+ AND k = ?
1589
+ AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
1590
+ }
1591
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1592
+ FROM vec_chat_messages v
1593
+ JOIN chat_messages c ON c.id = v.chat_message_id
1594
+ WHERE v.embedding MATCH ?
1595
+ AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
1596
+ }
1527
1597
  getTranscriptChatMessage(sessionId, transcriptIndex) {
1528
1598
  return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
1529
1599
  }
@@ -2587,6 +2657,9 @@ function composeEmbeddingText(obs) {
2587
2657
 
2588
2658
  `);
2589
2659
  }
2660
+ function composeChatEmbeddingText(text) {
2661
+ return text.replace(/\s+/g, " ").trim().slice(0, 2000);
2662
+ }
2590
2663
  async function getPipeline() {
2591
2664
  if (_pipeline)
2592
2665
  return _pipeline;
@@ -3954,7 +4027,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
3954
4027
  }
3955
4028
  return messages;
3956
4029
  }
3957
- function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
4030
+ async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
3958
4031
  const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
3959
4032
  ...message,
3960
4033
  text: message.text.trim()
@@ -3984,6 +4057,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
3984
4057
  transcript_index: transcriptIndex
3985
4058
  });
3986
4059
  db.addToOutbox("chat_message", row.id);
4060
+ if (db.vecAvailable) {
4061
+ const embedding = await embedText(composeChatEmbeddingText(message.text));
4062
+ if (embedding) {
4063
+ db.vecChatInsert(row.id, embedding);
4064
+ }
4065
+ }
3987
4066
  imported++;
3988
4067
  }
3989
4068
  return { imported, total: messages.length };
@@ -4117,7 +4196,7 @@ async function main() {
4117
4196
  try {
4118
4197
  let importedChat = 0;
4119
4198
  if (event.session_id) {
4120
- const chatSync = syncTranscriptChat(db, config, event.session_id, event.cwd);
4199
+ const chatSync = await syncTranscriptChat(db, config, event.session_id, event.cwd);
4121
4200
  importedChat = chatSync.imported;
4122
4201
  await upsertRollingHandoff(db, config, {
4123
4202
  session_id: event.session_id,
@@ -756,6 +756,17 @@ var MIGRATIONS = [
756
756
  ON chat_messages(session_id, transcript_index)
757
757
  WHERE transcript_index IS NOT NULL;
758
758
  `
759
+ },
760
+ {
761
+ version: 18,
762
+ description: "Add sqlite-vec semantic search for chat recall",
763
+ condition: (db) => isVecExtensionLoaded(db),
764
+ sql: `
765
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
766
+ chat_message_id INTEGER PRIMARY KEY,
767
+ embedding FLOAT[384]
768
+ );
769
+ `
759
770
  }
760
771
  ];
761
772
  function isVecExtensionLoaded(db) {
@@ -829,6 +840,9 @@ function inferLegacySchemaVersion(db) {
829
840
  if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
830
841
  version = Math.max(version, 17);
831
842
  }
843
+ if (tableExists(db, "vec_chat_messages")) {
844
+ version = Math.max(version, 18);
845
+ }
832
846
  return version;
833
847
  }
834
848
  function runMigrations(db) {
@@ -945,6 +959,20 @@ function ensureChatMessageColumns(db) {
945
959
  db.exec("PRAGMA user_version = 17");
946
960
  }
947
961
  }
962
+ function ensureChatVectorTable(db) {
963
+ if (!isVecExtensionLoaded(db))
964
+ return;
965
+ db.exec(`
966
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
967
+ chat_message_id INTEGER PRIMARY KEY,
968
+ embedding FLOAT[384]
969
+ );
970
+ `);
971
+ const current = getSchemaVersion(db);
972
+ if (current < 18) {
973
+ db.exec("PRAGMA user_version = 18");
974
+ }
975
+ }
948
976
  function ensureSyncOutboxSupportsChatMessages(db) {
949
977
  if (syncOutboxSupportsChatMessages(db)) {
950
978
  const current = getSchemaVersion(db);
@@ -1158,6 +1186,7 @@ class MemDatabase {
1158
1186
  ensureObservationTypes(this.db);
1159
1187
  ensureSessionSummaryColumns(this.db);
1160
1188
  ensureChatMessageColumns(this.db);
1189
+ ensureChatVectorTable(this.db);
1161
1190
  ensureSyncOutboxSupportsChatMessages(this.db);
1162
1191
  }
1163
1192
  loadVecExtension() {
@@ -1524,6 +1553,14 @@ class MemDatabase {
1524
1553
  getChatMessageByRemoteSourceId(remoteSourceId) {
1525
1554
  return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
1526
1555
  }
1556
+ getChatMessagesByIds(ids) {
1557
+ if (ids.length === 0)
1558
+ return [];
1559
+ const placeholders = ids.map(() => "?").join(",");
1560
+ const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
1561
+ const order = new Map(ids.map((id, index) => [id, index]));
1562
+ return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
1563
+ }
1527
1564
  getSessionChatMessages(sessionId, limit = 50) {
1528
1565
  return this.db.query(`SELECT * FROM chat_messages
1529
1566
  WHERE session_id = ?
@@ -1600,6 +1637,39 @@ class MemDatabase {
1600
1637
  ORDER BY created_at_epoch DESC, id DESC
1601
1638
  LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
1602
1639
  }
1640
+ vecChatInsert(chatMessageId, embedding) {
1641
+ if (!this.vecAvailable)
1642
+ return;
1643
+ this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
1644
+ }
1645
+ searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
1646
+ if (!this.vecAvailable)
1647
+ return [];
1648
+ const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
1649
+ const visibilityClause = userId ? " AND c.user_id = ?" : "";
1650
+ const transcriptPreference = `
1651
+ AND (
1652
+ c.source_kind = 'transcript'
1653
+ OR NOT EXISTS (
1654
+ SELECT 1 FROM chat_messages t2
1655
+ WHERE t2.session_id = c.session_id
1656
+ AND t2.source_kind = 'transcript'
1657
+ )
1658
+ )`;
1659
+ if (projectId !== null) {
1660
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1661
+ FROM vec_chat_messages v
1662
+ JOIN chat_messages c ON c.id = v.chat_message_id
1663
+ WHERE v.embedding MATCH ?
1664
+ AND k = ?
1665
+ AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
1666
+ }
1667
+ return this.db.query(`SELECT v.chat_message_id, v.distance
1668
+ FROM vec_chat_messages v
1669
+ JOIN chat_messages c ON c.id = v.chat_message_id
1670
+ WHERE v.embedding MATCH ?
1671
+ AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
1672
+ }
1603
1673
  getTranscriptChatMessage(sessionId, transcriptIndex) {
1604
1674
  return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
1605
1675
  }