engrm 0.4.26 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/cli.js +84 -0
- package/dist/hooks/elicitation-result.js +73 -0
- package/dist/hooks/post-tool-use.js +89 -2
- package/dist/hooks/pre-compact.js +89 -2
- package/dist/hooks/sentinel.js +70 -0
- package/dist/hooks/session-start.js +92 -2
- package/dist/hooks/stop.js +96 -3
- package/dist/hooks/user-prompt-submit.js +154 -61
- package/dist/server.js +357 -59
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -227,7 +227,8 @@ The MCP server exposes tools that supported agents can call directly:
|
|
|
227
227
|
| `load_handoff` | Open a saved handoff as a resume point for a new session |
|
|
228
228
|
| `refresh_chat_recall` | Rehydrate the separate chat lane from a Claude transcript when a long session feels under-captured |
|
|
229
229
|
| `recent_chat` | Inspect the separate synced chat lane without mixing it into durable memory |
|
|
230
|
-
| `search_chat` | Search recent chat recall separately from reusable memory observations |
|
|
230
|
+
| `search_chat` | Search recent chat recall with hybrid lexical + semantic matching, separately from reusable memory observations |
|
|
231
|
+
| `search_recall` | Search durable memory and chat recall together when you do not want to guess the right lane |
|
|
231
232
|
| `plugin_catalog` | Inspect Engrm plugin manifests for memory-aware integrations |
|
|
232
233
|
| `save_plugin_memory` | Save reduced plugin output with stable Engrm provenance |
|
|
233
234
|
| `capture_git_diff` | Reduce a git diff into a durable memory object and save it |
|
|
@@ -359,6 +360,10 @@ What each tool is good for:
|
|
|
359
360
|
- `capture_status` tells you whether prompt/tool hooks are live on this machine
|
|
360
361
|
- `capture_quality` shows whether chat recall is transcript-backed or still hook-only across the workspace
|
|
361
362
|
- `memory_console` gives the quickest project snapshot, including whether continuity is `fresh`, `thin`, or `cold`
|
|
363
|
+
- `memory_console`, `project_memory_index`, and `session_context` now also show whether project chat recall is transcript-backed or only hook-captured
|
|
364
|
+
- when chat continuity is only hook-captured, the workbench and startup hints now prefer `refresh_chat_recall`
|
|
365
|
+
- the workbench and startup hints now also prefer `search_recall` as the first “what were we just talking about?” path when recent prompts/chat/observations exist
|
|
366
|
+
- `search_chat` now uses hybrid lexical + semantic ranking when sqlite-vec and local embeddings are available, so recent conversation recall is less dependent on exact wording
|
|
362
367
|
- `activity_feed` shows the merged chronology across prompts, tools, chat, handoffs, observations, and summaries
|
|
363
368
|
- `recent_sessions` helps you pick a session worth opening
|
|
364
369
|
- `session_story` reconstructs one session in detail, including handoffs and chat recall
|
|
@@ -366,7 +371,7 @@ What each tool is good for:
|
|
|
366
371
|
- `session_tool_memory` shows which tool calls in one session turned into reusable memory and which did not
|
|
367
372
|
- `project_memory_index` shows typed memory by repo, including continuity state and hot files
|
|
368
373
|
- `workspace_memory_index` shows coverage across all repos on the machine
|
|
369
|
-
- `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, so weak OpenClaw recall is easier to diagnose and refresh
|
|
374
|
+
- `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, and `search_chat` will also mark when semantic ranking was available, so weak OpenClaw recall is easier to diagnose and refresh
|
|
370
375
|
|
|
371
376
|
### Thin Tool Workflow
|
|
372
377
|
|
package/dist/cli.js
CHANGED
|
@@ -708,6 +708,17 @@ var MIGRATIONS = [
|
|
|
708
708
|
ON chat_messages(session_id, transcript_index)
|
|
709
709
|
WHERE transcript_index IS NOT NULL;
|
|
710
710
|
`
|
|
711
|
+
},
|
|
712
|
+
{
|
|
713
|
+
version: 18,
|
|
714
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
715
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
716
|
+
sql: `
|
|
717
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
718
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
719
|
+
embedding FLOAT[384]
|
|
720
|
+
);
|
|
721
|
+
`
|
|
711
722
|
}
|
|
712
723
|
];
|
|
713
724
|
function isVecExtensionLoaded(db) {
|
|
@@ -781,6 +792,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
781
792
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
782
793
|
version = Math.max(version, 17);
|
|
783
794
|
}
|
|
795
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
796
|
+
version = Math.max(version, 18);
|
|
797
|
+
}
|
|
784
798
|
return version;
|
|
785
799
|
}
|
|
786
800
|
function runMigrations(db) {
|
|
@@ -897,6 +911,20 @@ function ensureChatMessageColumns(db) {
|
|
|
897
911
|
db.exec("PRAGMA user_version = 17");
|
|
898
912
|
}
|
|
899
913
|
}
|
|
914
|
+
function ensureChatVectorTable(db) {
|
|
915
|
+
if (!isVecExtensionLoaded(db))
|
|
916
|
+
return;
|
|
917
|
+
db.exec(`
|
|
918
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
919
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
920
|
+
embedding FLOAT[384]
|
|
921
|
+
);
|
|
922
|
+
`);
|
|
923
|
+
const current = getSchemaVersion(db);
|
|
924
|
+
if (current < 18) {
|
|
925
|
+
db.exec("PRAGMA user_version = 18");
|
|
926
|
+
}
|
|
927
|
+
}
|
|
900
928
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
901
929
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
902
930
|
const current = getSchemaVersion(db);
|
|
@@ -1110,6 +1138,7 @@ class MemDatabase {
|
|
|
1110
1138
|
ensureObservationTypes(this.db);
|
|
1111
1139
|
ensureSessionSummaryColumns(this.db);
|
|
1112
1140
|
ensureChatMessageColumns(this.db);
|
|
1141
|
+
ensureChatVectorTable(this.db);
|
|
1113
1142
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1114
1143
|
}
|
|
1115
1144
|
loadVecExtension() {
|
|
@@ -1476,6 +1505,14 @@ class MemDatabase {
|
|
|
1476
1505
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1477
1506
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1478
1507
|
}
|
|
1508
|
+
getChatMessagesByIds(ids) {
|
|
1509
|
+
if (ids.length === 0)
|
|
1510
|
+
return [];
|
|
1511
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1512
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1513
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1514
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1515
|
+
}
|
|
1479
1516
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1480
1517
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1481
1518
|
WHERE session_id = ?
|
|
@@ -1552,6 +1589,39 @@ class MemDatabase {
|
|
|
1552
1589
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1553
1590
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1554
1591
|
}
|
|
1592
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1593
|
+
if (!this.vecAvailable)
|
|
1594
|
+
return;
|
|
1595
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1596
|
+
}
|
|
1597
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1598
|
+
if (!this.vecAvailable)
|
|
1599
|
+
return [];
|
|
1600
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1601
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1602
|
+
const transcriptPreference = `
|
|
1603
|
+
AND (
|
|
1604
|
+
c.source_kind = 'transcript'
|
|
1605
|
+
OR NOT EXISTS (
|
|
1606
|
+
SELECT 1 FROM chat_messages t2
|
|
1607
|
+
WHERE t2.session_id = c.session_id
|
|
1608
|
+
AND t2.source_kind = 'transcript'
|
|
1609
|
+
)
|
|
1610
|
+
)`;
|
|
1611
|
+
if (projectId !== null) {
|
|
1612
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1613
|
+
FROM vec_chat_messages v
|
|
1614
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1615
|
+
WHERE v.embedding MATCH ?
|
|
1616
|
+
AND k = ?
|
|
1617
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1618
|
+
}
|
|
1619
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1620
|
+
FROM vec_chat_messages v
|
|
1621
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1622
|
+
WHERE v.embedding MATCH ?
|
|
1623
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1624
|
+
}
|
|
1555
1625
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1556
1626
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1557
1627
|
}
|
|
@@ -2296,6 +2366,17 @@ var MIGRATIONS2 = [
|
|
|
2296
2366
|
ON chat_messages(session_id, transcript_index)
|
|
2297
2367
|
WHERE transcript_index IS NOT NULL;
|
|
2298
2368
|
`
|
|
2369
|
+
},
|
|
2370
|
+
{
|
|
2371
|
+
version: 18,
|
|
2372
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
2373
|
+
condition: (db) => isVecExtensionLoaded2(db),
|
|
2374
|
+
sql: `
|
|
2375
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
2376
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
2377
|
+
embedding FLOAT[384]
|
|
2378
|
+
);
|
|
2379
|
+
`
|
|
2299
2380
|
}
|
|
2300
2381
|
];
|
|
2301
2382
|
function isVecExtensionLoaded2(db) {
|
|
@@ -3288,6 +3369,9 @@ function composeEmbeddingText(obs) {
|
|
|
3288
3369
|
|
|
3289
3370
|
`);
|
|
3290
3371
|
}
|
|
3372
|
+
function composeChatEmbeddingText(text) {
|
|
3373
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
3374
|
+
}
|
|
3291
3375
|
async function getPipeline() {
|
|
3292
3376
|
if (_pipeline)
|
|
3293
3377
|
return _pipeline;
|
|
@@ -510,6 +510,9 @@ function composeEmbeddingText(obs) {
|
|
|
510
510
|
|
|
511
511
|
`);
|
|
512
512
|
}
|
|
513
|
+
function composeChatEmbeddingText(text) {
|
|
514
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
515
|
+
}
|
|
513
516
|
async function getPipeline() {
|
|
514
517
|
if (_pipeline)
|
|
515
518
|
return _pipeline;
|
|
@@ -1541,6 +1544,17 @@ var MIGRATIONS = [
|
|
|
1541
1544
|
ON chat_messages(session_id, transcript_index)
|
|
1542
1545
|
WHERE transcript_index IS NOT NULL;
|
|
1543
1546
|
`
|
|
1547
|
+
},
|
|
1548
|
+
{
|
|
1549
|
+
version: 18,
|
|
1550
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
1551
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
1552
|
+
sql: `
|
|
1553
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1554
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1555
|
+
embedding FLOAT[384]
|
|
1556
|
+
);
|
|
1557
|
+
`
|
|
1544
1558
|
}
|
|
1545
1559
|
];
|
|
1546
1560
|
function isVecExtensionLoaded(db) {
|
|
@@ -1614,6 +1628,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
1614
1628
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
1615
1629
|
version = Math.max(version, 17);
|
|
1616
1630
|
}
|
|
1631
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
1632
|
+
version = Math.max(version, 18);
|
|
1633
|
+
}
|
|
1617
1634
|
return version;
|
|
1618
1635
|
}
|
|
1619
1636
|
function runMigrations(db) {
|
|
@@ -1730,6 +1747,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1730
1747
|
db.exec("PRAGMA user_version = 17");
|
|
1731
1748
|
}
|
|
1732
1749
|
}
|
|
1750
|
+
function ensureChatVectorTable(db) {
|
|
1751
|
+
if (!isVecExtensionLoaded(db))
|
|
1752
|
+
return;
|
|
1753
|
+
db.exec(`
|
|
1754
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1755
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1756
|
+
embedding FLOAT[384]
|
|
1757
|
+
);
|
|
1758
|
+
`);
|
|
1759
|
+
const current = getSchemaVersion(db);
|
|
1760
|
+
if (current < 18) {
|
|
1761
|
+
db.exec("PRAGMA user_version = 18");
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1733
1764
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1734
1765
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1735
1766
|
const current = getSchemaVersion(db);
|
|
@@ -1943,6 +1974,7 @@ class MemDatabase {
|
|
|
1943
1974
|
ensureObservationTypes(this.db);
|
|
1944
1975
|
ensureSessionSummaryColumns(this.db);
|
|
1945
1976
|
ensureChatMessageColumns(this.db);
|
|
1977
|
+
ensureChatVectorTable(this.db);
|
|
1946
1978
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1947
1979
|
}
|
|
1948
1980
|
loadVecExtension() {
|
|
@@ -2309,6 +2341,14 @@ class MemDatabase {
|
|
|
2309
2341
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
2310
2342
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
2311
2343
|
}
|
|
2344
|
+
getChatMessagesByIds(ids) {
|
|
2345
|
+
if (ids.length === 0)
|
|
2346
|
+
return [];
|
|
2347
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
2348
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
2349
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
2350
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
2351
|
+
}
|
|
2312
2352
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
2313
2353
|
return this.db.query(`SELECT * FROM chat_messages
|
|
2314
2354
|
WHERE session_id = ?
|
|
@@ -2385,6 +2425,39 @@ class MemDatabase {
|
|
|
2385
2425
|
ORDER BY created_at_epoch DESC, id DESC
|
|
2386
2426
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
2387
2427
|
}
|
|
2428
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
2429
|
+
if (!this.vecAvailable)
|
|
2430
|
+
return;
|
|
2431
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
2432
|
+
}
|
|
2433
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
2434
|
+
if (!this.vecAvailable)
|
|
2435
|
+
return [];
|
|
2436
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
2437
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
2438
|
+
const transcriptPreference = `
|
|
2439
|
+
AND (
|
|
2440
|
+
c.source_kind = 'transcript'
|
|
2441
|
+
OR NOT EXISTS (
|
|
2442
|
+
SELECT 1 FROM chat_messages t2
|
|
2443
|
+
WHERE t2.session_id = c.session_id
|
|
2444
|
+
AND t2.source_kind = 'transcript'
|
|
2445
|
+
)
|
|
2446
|
+
)`;
|
|
2447
|
+
if (projectId !== null) {
|
|
2448
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
2449
|
+
FROM vec_chat_messages v
|
|
2450
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
2451
|
+
WHERE v.embedding MATCH ?
|
|
2452
|
+
AND k = ?
|
|
2453
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
2454
|
+
}
|
|
2455
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
2456
|
+
FROM vec_chat_messages v
|
|
2457
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
2458
|
+
WHERE v.embedding MATCH ?
|
|
2459
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
2460
|
+
}
|
|
2388
2461
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
2389
2462
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
2390
2463
|
}
|
|
@@ -886,6 +886,17 @@ var MIGRATIONS = [
|
|
|
886
886
|
ON chat_messages(session_id, transcript_index)
|
|
887
887
|
WHERE transcript_index IS NOT NULL;
|
|
888
888
|
`
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
version: 18,
|
|
892
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
893
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
894
|
+
sql: `
|
|
895
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
896
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
897
|
+
embedding FLOAT[384]
|
|
898
|
+
);
|
|
899
|
+
`
|
|
889
900
|
}
|
|
890
901
|
];
|
|
891
902
|
function isVecExtensionLoaded(db) {
|
|
@@ -959,6 +970,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
959
970
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
960
971
|
version = Math.max(version, 17);
|
|
961
972
|
}
|
|
973
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
974
|
+
version = Math.max(version, 18);
|
|
975
|
+
}
|
|
962
976
|
return version;
|
|
963
977
|
}
|
|
964
978
|
function runMigrations(db) {
|
|
@@ -1075,6 +1089,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1075
1089
|
db.exec("PRAGMA user_version = 17");
|
|
1076
1090
|
}
|
|
1077
1091
|
}
|
|
1092
|
+
function ensureChatVectorTable(db) {
|
|
1093
|
+
if (!isVecExtensionLoaded(db))
|
|
1094
|
+
return;
|
|
1095
|
+
db.exec(`
|
|
1096
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1097
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1098
|
+
embedding FLOAT[384]
|
|
1099
|
+
);
|
|
1100
|
+
`);
|
|
1101
|
+
const current = getSchemaVersion(db);
|
|
1102
|
+
if (current < 18) {
|
|
1103
|
+
db.exec("PRAGMA user_version = 18");
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1078
1106
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1079
1107
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1080
1108
|
const current = getSchemaVersion(db);
|
|
@@ -1288,6 +1316,7 @@ class MemDatabase {
|
|
|
1288
1316
|
ensureObservationTypes(this.db);
|
|
1289
1317
|
ensureSessionSummaryColumns(this.db);
|
|
1290
1318
|
ensureChatMessageColumns(this.db);
|
|
1319
|
+
ensureChatVectorTable(this.db);
|
|
1291
1320
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1292
1321
|
}
|
|
1293
1322
|
loadVecExtension() {
|
|
@@ -1654,6 +1683,14 @@ class MemDatabase {
|
|
|
1654
1683
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1655
1684
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1656
1685
|
}
|
|
1686
|
+
getChatMessagesByIds(ids) {
|
|
1687
|
+
if (ids.length === 0)
|
|
1688
|
+
return [];
|
|
1689
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1690
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1691
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1692
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1693
|
+
}
|
|
1657
1694
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1658
1695
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1659
1696
|
WHERE session_id = ?
|
|
@@ -1730,6 +1767,39 @@ class MemDatabase {
|
|
|
1730
1767
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1731
1768
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1732
1769
|
}
|
|
1770
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1771
|
+
if (!this.vecAvailable)
|
|
1772
|
+
return;
|
|
1773
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1774
|
+
}
|
|
1775
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1776
|
+
if (!this.vecAvailable)
|
|
1777
|
+
return [];
|
|
1778
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1779
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1780
|
+
const transcriptPreference = `
|
|
1781
|
+
AND (
|
|
1782
|
+
c.source_kind = 'transcript'
|
|
1783
|
+
OR NOT EXISTS (
|
|
1784
|
+
SELECT 1 FROM chat_messages t2
|
|
1785
|
+
WHERE t2.session_id = c.session_id
|
|
1786
|
+
AND t2.source_kind = 'transcript'
|
|
1787
|
+
)
|
|
1788
|
+
)`;
|
|
1789
|
+
if (projectId !== null) {
|
|
1790
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1791
|
+
FROM vec_chat_messages v
|
|
1792
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1793
|
+
WHERE v.embedding MATCH ?
|
|
1794
|
+
AND k = ?
|
|
1795
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1796
|
+
}
|
|
1797
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1798
|
+
FROM vec_chat_messages v
|
|
1799
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1800
|
+
WHERE v.embedding MATCH ?
|
|
1801
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1802
|
+
}
|
|
1733
1803
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1734
1804
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1735
1805
|
}
|
|
@@ -2514,6 +2584,9 @@ function composeEmbeddingText(obs) {
|
|
|
2514
2584
|
|
|
2515
2585
|
`);
|
|
2516
2586
|
}
|
|
2587
|
+
function composeChatEmbeddingText(text) {
|
|
2588
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2589
|
+
}
|
|
2517
2590
|
async function getPipeline() {
|
|
2518
2591
|
if (_pipeline)
|
|
2519
2592
|
return _pipeline;
|
|
@@ -3701,7 +3774,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
3701
3774
|
}
|
|
3702
3775
|
return messages;
|
|
3703
3776
|
}
|
|
3704
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3777
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3705
3778
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
3706
3779
|
...message,
|
|
3707
3780
|
text: message.text.trim()
|
|
@@ -3731,6 +3804,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
3731
3804
|
transcript_index: transcriptIndex
|
|
3732
3805
|
});
|
|
3733
3806
|
db.addToOutbox("chat_message", row.id);
|
|
3807
|
+
if (db.vecAvailable) {
|
|
3808
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
3809
|
+
if (embedding) {
|
|
3810
|
+
db.vecChatInsert(row.id, embedding);
|
|
3811
|
+
}
|
|
3812
|
+
}
|
|
3734
3813
|
imported++;
|
|
3735
3814
|
}
|
|
3736
3815
|
return { imported, total: messages.length };
|
|
@@ -3831,6 +3910,8 @@ function getSessionStory(db, input) {
|
|
|
3831
3910
|
summary,
|
|
3832
3911
|
prompts,
|
|
3833
3912
|
chat_messages: chatMessages,
|
|
3913
|
+
chat_source_summary: summarizeChatSources(chatMessages),
|
|
3914
|
+
chat_coverage_state: chatMessages.some((message) => message.source_kind === "transcript") ? "transcript-backed" : chatMessages.length > 0 ? "hook-only" : "none",
|
|
3834
3915
|
tool_events: toolEvents,
|
|
3835
3916
|
observations,
|
|
3836
3917
|
handoffs,
|
|
@@ -3928,6 +4009,12 @@ function collectProvenanceSummary(observations) {
|
|
|
3928
4009
|
}
|
|
3929
4010
|
return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => b.count - a.count || a.tool.localeCompare(b.tool)).slice(0, 6);
|
|
3930
4011
|
}
|
|
4012
|
+
function summarizeChatSources(messages) {
|
|
4013
|
+
return messages.reduce((summary, message) => {
|
|
4014
|
+
summary[message.source_kind] += 1;
|
|
4015
|
+
return summary;
|
|
4016
|
+
}, { transcript: 0, hook: 0 });
|
|
4017
|
+
}
|
|
3931
4018
|
|
|
3932
4019
|
// src/tools/handoffs.ts
|
|
3933
4020
|
async function upsertRollingHandoff(db, config, input) {
|
|
@@ -4248,7 +4335,7 @@ async function main() {
|
|
|
4248
4335
|
try {
|
|
4249
4336
|
if (event.session_id) {
|
|
4250
4337
|
persistRawToolChronology(event, config.user_id, config.device_id);
|
|
4251
|
-
syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4338
|
+
await syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4252
4339
|
}
|
|
4253
4340
|
const textToScan = extractScanText(event);
|
|
4254
4341
|
if (textToScan) {
|
|
@@ -680,6 +680,17 @@ var MIGRATIONS = [
|
|
|
680
680
|
ON chat_messages(session_id, transcript_index)
|
|
681
681
|
WHERE transcript_index IS NOT NULL;
|
|
682
682
|
`
|
|
683
|
+
},
|
|
684
|
+
{
|
|
685
|
+
version: 18,
|
|
686
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
687
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
688
|
+
sql: `
|
|
689
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
690
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
691
|
+
embedding FLOAT[384]
|
|
692
|
+
);
|
|
693
|
+
`
|
|
683
694
|
}
|
|
684
695
|
];
|
|
685
696
|
function isVecExtensionLoaded(db) {
|
|
@@ -753,6 +764,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
753
764
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
754
765
|
version = Math.max(version, 17);
|
|
755
766
|
}
|
|
767
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
768
|
+
version = Math.max(version, 18);
|
|
769
|
+
}
|
|
756
770
|
return version;
|
|
757
771
|
}
|
|
758
772
|
function runMigrations(db) {
|
|
@@ -869,6 +883,20 @@ function ensureChatMessageColumns(db) {
|
|
|
869
883
|
db.exec("PRAGMA user_version = 17");
|
|
870
884
|
}
|
|
871
885
|
}
|
|
886
|
+
function ensureChatVectorTable(db) {
|
|
887
|
+
if (!isVecExtensionLoaded(db))
|
|
888
|
+
return;
|
|
889
|
+
db.exec(`
|
|
890
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
891
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
892
|
+
embedding FLOAT[384]
|
|
893
|
+
);
|
|
894
|
+
`);
|
|
895
|
+
const current = getSchemaVersion(db);
|
|
896
|
+
if (current < 18) {
|
|
897
|
+
db.exec("PRAGMA user_version = 18");
|
|
898
|
+
}
|
|
899
|
+
}
|
|
872
900
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
873
901
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
874
902
|
const current = getSchemaVersion(db);
|
|
@@ -1082,6 +1110,7 @@ class MemDatabase {
|
|
|
1082
1110
|
ensureObservationTypes(this.db);
|
|
1083
1111
|
ensureSessionSummaryColumns(this.db);
|
|
1084
1112
|
ensureChatMessageColumns(this.db);
|
|
1113
|
+
ensureChatVectorTable(this.db);
|
|
1085
1114
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1086
1115
|
}
|
|
1087
1116
|
loadVecExtension() {
|
|
@@ -1448,6 +1477,14 @@ class MemDatabase {
|
|
|
1448
1477
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1449
1478
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1450
1479
|
}
|
|
1480
|
+
getChatMessagesByIds(ids) {
|
|
1481
|
+
if (ids.length === 0)
|
|
1482
|
+
return [];
|
|
1483
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1484
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1485
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1486
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1487
|
+
}
|
|
1451
1488
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1452
1489
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1453
1490
|
WHERE session_id = ?
|
|
@@ -1524,6 +1561,39 @@ class MemDatabase {
|
|
|
1524
1561
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1525
1562
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1526
1563
|
}
|
|
1564
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1565
|
+
if (!this.vecAvailable)
|
|
1566
|
+
return;
|
|
1567
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1568
|
+
}
|
|
1569
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1570
|
+
if (!this.vecAvailable)
|
|
1571
|
+
return [];
|
|
1572
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1573
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1574
|
+
const transcriptPreference = `
|
|
1575
|
+
AND (
|
|
1576
|
+
c.source_kind = 'transcript'
|
|
1577
|
+
OR NOT EXISTS (
|
|
1578
|
+
SELECT 1 FROM chat_messages t2
|
|
1579
|
+
WHERE t2.session_id = c.session_id
|
|
1580
|
+
AND t2.source_kind = 'transcript'
|
|
1581
|
+
)
|
|
1582
|
+
)`;
|
|
1583
|
+
if (projectId !== null) {
|
|
1584
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1585
|
+
FROM vec_chat_messages v
|
|
1586
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1587
|
+
WHERE v.embedding MATCH ?
|
|
1588
|
+
AND k = ?
|
|
1589
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1590
|
+
}
|
|
1591
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1592
|
+
FROM vec_chat_messages v
|
|
1593
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1594
|
+
WHERE v.embedding MATCH ?
|
|
1595
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1596
|
+
}
|
|
1527
1597
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1528
1598
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1529
1599
|
}
|
|
@@ -2154,6 +2224,8 @@ function getSessionStory(db, input) {
|
|
|
2154
2224
|
summary,
|
|
2155
2225
|
prompts,
|
|
2156
2226
|
chat_messages: chatMessages,
|
|
2227
|
+
chat_source_summary: summarizeChatSources(chatMessages),
|
|
2228
|
+
chat_coverage_state: chatMessages.some((message) => message.source_kind === "transcript") ? "transcript-backed" : chatMessages.length > 0 ? "hook-only" : "none",
|
|
2157
2229
|
tool_events: toolEvents,
|
|
2158
2230
|
observations,
|
|
2159
2231
|
handoffs,
|
|
@@ -2251,6 +2323,12 @@ function collectProvenanceSummary(observations) {
|
|
|
2251
2323
|
}
|
|
2252
2324
|
return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => b.count - a.count || a.tool.localeCompare(b.tool)).slice(0, 6);
|
|
2253
2325
|
}
|
|
2326
|
+
function summarizeChatSources(messages) {
|
|
2327
|
+
return messages.reduce((summary, message) => {
|
|
2328
|
+
summary[message.source_kind] += 1;
|
|
2329
|
+
return summary;
|
|
2330
|
+
}, { transcript: 0, hook: 0 });
|
|
2331
|
+
}
|
|
2254
2332
|
|
|
2255
2333
|
// src/tools/save.ts
|
|
2256
2334
|
import { relative, isAbsolute } from "node:path";
|
|
@@ -2579,6 +2657,9 @@ function composeEmbeddingText(obs) {
|
|
|
2579
2657
|
|
|
2580
2658
|
`);
|
|
2581
2659
|
}
|
|
2660
|
+
function composeChatEmbeddingText(text) {
|
|
2661
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2662
|
+
}
|
|
2582
2663
|
async function getPipeline() {
|
|
2583
2664
|
if (_pipeline)
|
|
2584
2665
|
return _pipeline;
|
|
@@ -3946,7 +4027,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
3946
4027
|
}
|
|
3947
4028
|
return messages;
|
|
3948
4029
|
}
|
|
3949
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
4030
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3950
4031
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
3951
4032
|
...message,
|
|
3952
4033
|
text: message.text.trim()
|
|
@@ -3976,6 +4057,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
3976
4057
|
transcript_index: transcriptIndex
|
|
3977
4058
|
});
|
|
3978
4059
|
db.addToOutbox("chat_message", row.id);
|
|
4060
|
+
if (db.vecAvailable) {
|
|
4061
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
4062
|
+
if (embedding) {
|
|
4063
|
+
db.vecChatInsert(row.id, embedding);
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
3979
4066
|
imported++;
|
|
3980
4067
|
}
|
|
3981
4068
|
return { imported, total: messages.length };
|
|
@@ -4109,7 +4196,7 @@ async function main() {
|
|
|
4109
4196
|
try {
|
|
4110
4197
|
let importedChat = 0;
|
|
4111
4198
|
if (event.session_id) {
|
|
4112
|
-
const chatSync = syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4199
|
+
const chatSync = await syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4113
4200
|
importedChat = chatSync.imported;
|
|
4114
4201
|
await upsertRollingHandoff(db, config, {
|
|
4115
4202
|
session_id: event.session_id,
|