engrm 0.4.27 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/cli.js +84 -0
- package/dist/hooks/elicitation-result.js +73 -0
- package/dist/hooks/post-tool-use.js +81 -2
- package/dist/hooks/pre-compact.js +81 -2
- package/dist/hooks/sentinel.js +70 -0
- package/dist/hooks/session-start.js +77 -1
- package/dist/hooks/stop.js +88 -3
- package/dist/hooks/user-prompt-submit.js +146 -61
- package/dist/server.js +121 -10
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -227,7 +227,7 @@ The MCP server exposes tools that supported agents can call directly:
|
|
|
227
227
|
| `load_handoff` | Open a saved handoff as a resume point for a new session |
|
|
228
228
|
| `refresh_chat_recall` | Rehydrate the separate chat lane from a Claude transcript when a long session feels under-captured |
|
|
229
229
|
| `recent_chat` | Inspect the separate synced chat lane without mixing it into durable memory |
|
|
230
|
-
| `search_chat` | Search recent chat recall separately from reusable memory observations |
|
|
230
|
+
| `search_chat` | Search recent chat recall with hybrid lexical + semantic matching, separately from reusable memory observations |
|
|
231
231
|
| `search_recall` | Search durable memory and chat recall together when you do not want to guess the right lane |
|
|
232
232
|
| `plugin_catalog` | Inspect Engrm plugin manifests for memory-aware integrations |
|
|
233
233
|
| `save_plugin_memory` | Save reduced plugin output with stable Engrm provenance |
|
|
@@ -362,6 +362,8 @@ What each tool is good for:
|
|
|
362
362
|
- `memory_console` gives the quickest project snapshot, including whether continuity is `fresh`, `thin`, or `cold`
|
|
363
363
|
- `memory_console`, `project_memory_index`, and `session_context` now also show whether project chat recall is transcript-backed or only hook-captured
|
|
364
364
|
- when chat continuity is only hook-captured, the workbench and startup hints now prefer `refresh_chat_recall`
|
|
365
|
+
- the workbench and startup hints now also prefer `search_recall` as the first “what were we just talking about?” path when recent prompts/chat/observations exist
|
|
366
|
+
- `search_chat` now uses hybrid lexical + semantic ranking when sqlite-vec and local embeddings are available, so recent conversation recall is less dependent on exact wording
|
|
365
367
|
- `activity_feed` shows the merged chronology across prompts, tools, chat, handoffs, observations, and summaries
|
|
366
368
|
- `recent_sessions` helps you pick a session worth opening
|
|
367
369
|
- `session_story` reconstructs one session in detail, including handoffs and chat recall
|
|
@@ -369,7 +371,7 @@ What each tool is good for:
|
|
|
369
371
|
- `session_tool_memory` shows which tool calls in one session turned into reusable memory and which did not
|
|
370
372
|
- `project_memory_index` shows typed memory by repo, including continuity state and hot files
|
|
371
373
|
- `workspace_memory_index` shows coverage across all repos on the machine
|
|
372
|
-
- `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, so weak OpenClaw recall is easier to diagnose and refresh
|
|
374
|
+
- `recent_chat` / `search_chat` now report transcript-vs-hook coverage too, and `search_chat` will also mark when semantic ranking was available, so weak OpenClaw recall is easier to diagnose and refresh
|
|
373
375
|
|
|
374
376
|
### Thin Tool Workflow
|
|
375
377
|
|
package/dist/cli.js
CHANGED
|
@@ -708,6 +708,17 @@ var MIGRATIONS = [
|
|
|
708
708
|
ON chat_messages(session_id, transcript_index)
|
|
709
709
|
WHERE transcript_index IS NOT NULL;
|
|
710
710
|
`
|
|
711
|
+
},
|
|
712
|
+
{
|
|
713
|
+
version: 18,
|
|
714
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
715
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
716
|
+
sql: `
|
|
717
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
718
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
719
|
+
embedding FLOAT[384]
|
|
720
|
+
);
|
|
721
|
+
`
|
|
711
722
|
}
|
|
712
723
|
];
|
|
713
724
|
function isVecExtensionLoaded(db) {
|
|
@@ -781,6 +792,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
781
792
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
782
793
|
version = Math.max(version, 17);
|
|
783
794
|
}
|
|
795
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
796
|
+
version = Math.max(version, 18);
|
|
797
|
+
}
|
|
784
798
|
return version;
|
|
785
799
|
}
|
|
786
800
|
function runMigrations(db) {
|
|
@@ -897,6 +911,20 @@ function ensureChatMessageColumns(db) {
|
|
|
897
911
|
db.exec("PRAGMA user_version = 17");
|
|
898
912
|
}
|
|
899
913
|
}
|
|
914
|
+
function ensureChatVectorTable(db) {
|
|
915
|
+
if (!isVecExtensionLoaded(db))
|
|
916
|
+
return;
|
|
917
|
+
db.exec(`
|
|
918
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
919
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
920
|
+
embedding FLOAT[384]
|
|
921
|
+
);
|
|
922
|
+
`);
|
|
923
|
+
const current = getSchemaVersion(db);
|
|
924
|
+
if (current < 18) {
|
|
925
|
+
db.exec("PRAGMA user_version = 18");
|
|
926
|
+
}
|
|
927
|
+
}
|
|
900
928
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
901
929
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
902
930
|
const current = getSchemaVersion(db);
|
|
@@ -1110,6 +1138,7 @@ class MemDatabase {
|
|
|
1110
1138
|
ensureObservationTypes(this.db);
|
|
1111
1139
|
ensureSessionSummaryColumns(this.db);
|
|
1112
1140
|
ensureChatMessageColumns(this.db);
|
|
1141
|
+
ensureChatVectorTable(this.db);
|
|
1113
1142
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1114
1143
|
}
|
|
1115
1144
|
loadVecExtension() {
|
|
@@ -1476,6 +1505,14 @@ class MemDatabase {
|
|
|
1476
1505
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1477
1506
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1478
1507
|
}
|
|
1508
|
+
getChatMessagesByIds(ids) {
|
|
1509
|
+
if (ids.length === 0)
|
|
1510
|
+
return [];
|
|
1511
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1512
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1513
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1514
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1515
|
+
}
|
|
1479
1516
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1480
1517
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1481
1518
|
WHERE session_id = ?
|
|
@@ -1552,6 +1589,39 @@ class MemDatabase {
|
|
|
1552
1589
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1553
1590
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1554
1591
|
}
|
|
1592
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1593
|
+
if (!this.vecAvailable)
|
|
1594
|
+
return;
|
|
1595
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1596
|
+
}
|
|
1597
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1598
|
+
if (!this.vecAvailable)
|
|
1599
|
+
return [];
|
|
1600
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1601
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1602
|
+
const transcriptPreference = `
|
|
1603
|
+
AND (
|
|
1604
|
+
c.source_kind = 'transcript'
|
|
1605
|
+
OR NOT EXISTS (
|
|
1606
|
+
SELECT 1 FROM chat_messages t2
|
|
1607
|
+
WHERE t2.session_id = c.session_id
|
|
1608
|
+
AND t2.source_kind = 'transcript'
|
|
1609
|
+
)
|
|
1610
|
+
)`;
|
|
1611
|
+
if (projectId !== null) {
|
|
1612
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1613
|
+
FROM vec_chat_messages v
|
|
1614
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1615
|
+
WHERE v.embedding MATCH ?
|
|
1616
|
+
AND k = ?
|
|
1617
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1618
|
+
}
|
|
1619
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1620
|
+
FROM vec_chat_messages v
|
|
1621
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1622
|
+
WHERE v.embedding MATCH ?
|
|
1623
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1624
|
+
}
|
|
1555
1625
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1556
1626
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1557
1627
|
}
|
|
@@ -2296,6 +2366,17 @@ var MIGRATIONS2 = [
|
|
|
2296
2366
|
ON chat_messages(session_id, transcript_index)
|
|
2297
2367
|
WHERE transcript_index IS NOT NULL;
|
|
2298
2368
|
`
|
|
2369
|
+
},
|
|
2370
|
+
{
|
|
2371
|
+
version: 18,
|
|
2372
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
2373
|
+
condition: (db) => isVecExtensionLoaded2(db),
|
|
2374
|
+
sql: `
|
|
2375
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
2376
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
2377
|
+
embedding FLOAT[384]
|
|
2378
|
+
);
|
|
2379
|
+
`
|
|
2299
2380
|
}
|
|
2300
2381
|
];
|
|
2301
2382
|
function isVecExtensionLoaded2(db) {
|
|
@@ -3288,6 +3369,9 @@ function composeEmbeddingText(obs) {
|
|
|
3288
3369
|
|
|
3289
3370
|
`);
|
|
3290
3371
|
}
|
|
3372
|
+
function composeChatEmbeddingText(text) {
|
|
3373
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
3374
|
+
}
|
|
3291
3375
|
async function getPipeline() {
|
|
3292
3376
|
if (_pipeline)
|
|
3293
3377
|
return _pipeline;
|
|
@@ -510,6 +510,9 @@ function composeEmbeddingText(obs) {
|
|
|
510
510
|
|
|
511
511
|
`);
|
|
512
512
|
}
|
|
513
|
+
function composeChatEmbeddingText(text) {
|
|
514
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
515
|
+
}
|
|
513
516
|
async function getPipeline() {
|
|
514
517
|
if (_pipeline)
|
|
515
518
|
return _pipeline;
|
|
@@ -1541,6 +1544,17 @@ var MIGRATIONS = [
|
|
|
1541
1544
|
ON chat_messages(session_id, transcript_index)
|
|
1542
1545
|
WHERE transcript_index IS NOT NULL;
|
|
1543
1546
|
`
|
|
1547
|
+
},
|
|
1548
|
+
{
|
|
1549
|
+
version: 18,
|
|
1550
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
1551
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
1552
|
+
sql: `
|
|
1553
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1554
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1555
|
+
embedding FLOAT[384]
|
|
1556
|
+
);
|
|
1557
|
+
`
|
|
1544
1558
|
}
|
|
1545
1559
|
];
|
|
1546
1560
|
function isVecExtensionLoaded(db) {
|
|
@@ -1614,6 +1628,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
1614
1628
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
1615
1629
|
version = Math.max(version, 17);
|
|
1616
1630
|
}
|
|
1631
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
1632
|
+
version = Math.max(version, 18);
|
|
1633
|
+
}
|
|
1617
1634
|
return version;
|
|
1618
1635
|
}
|
|
1619
1636
|
function runMigrations(db) {
|
|
@@ -1730,6 +1747,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1730
1747
|
db.exec("PRAGMA user_version = 17");
|
|
1731
1748
|
}
|
|
1732
1749
|
}
|
|
1750
|
+
function ensureChatVectorTable(db) {
|
|
1751
|
+
if (!isVecExtensionLoaded(db))
|
|
1752
|
+
return;
|
|
1753
|
+
db.exec(`
|
|
1754
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1755
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1756
|
+
embedding FLOAT[384]
|
|
1757
|
+
);
|
|
1758
|
+
`);
|
|
1759
|
+
const current = getSchemaVersion(db);
|
|
1760
|
+
if (current < 18) {
|
|
1761
|
+
db.exec("PRAGMA user_version = 18");
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1733
1764
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1734
1765
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1735
1766
|
const current = getSchemaVersion(db);
|
|
@@ -1943,6 +1974,7 @@ class MemDatabase {
|
|
|
1943
1974
|
ensureObservationTypes(this.db);
|
|
1944
1975
|
ensureSessionSummaryColumns(this.db);
|
|
1945
1976
|
ensureChatMessageColumns(this.db);
|
|
1977
|
+
ensureChatVectorTable(this.db);
|
|
1946
1978
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1947
1979
|
}
|
|
1948
1980
|
loadVecExtension() {
|
|
@@ -2309,6 +2341,14 @@ class MemDatabase {
|
|
|
2309
2341
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
2310
2342
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
2311
2343
|
}
|
|
2344
|
+
getChatMessagesByIds(ids) {
|
|
2345
|
+
if (ids.length === 0)
|
|
2346
|
+
return [];
|
|
2347
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
2348
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
2349
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
2350
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
2351
|
+
}
|
|
2312
2352
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
2313
2353
|
return this.db.query(`SELECT * FROM chat_messages
|
|
2314
2354
|
WHERE session_id = ?
|
|
@@ -2385,6 +2425,39 @@ class MemDatabase {
|
|
|
2385
2425
|
ORDER BY created_at_epoch DESC, id DESC
|
|
2386
2426
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
2387
2427
|
}
|
|
2428
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
2429
|
+
if (!this.vecAvailable)
|
|
2430
|
+
return;
|
|
2431
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
2432
|
+
}
|
|
2433
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
2434
|
+
if (!this.vecAvailable)
|
|
2435
|
+
return [];
|
|
2436
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
2437
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
2438
|
+
const transcriptPreference = `
|
|
2439
|
+
AND (
|
|
2440
|
+
c.source_kind = 'transcript'
|
|
2441
|
+
OR NOT EXISTS (
|
|
2442
|
+
SELECT 1 FROM chat_messages t2
|
|
2443
|
+
WHERE t2.session_id = c.session_id
|
|
2444
|
+
AND t2.source_kind = 'transcript'
|
|
2445
|
+
)
|
|
2446
|
+
)`;
|
|
2447
|
+
if (projectId !== null) {
|
|
2448
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
2449
|
+
FROM vec_chat_messages v
|
|
2450
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
2451
|
+
WHERE v.embedding MATCH ?
|
|
2452
|
+
AND k = ?
|
|
2453
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
2454
|
+
}
|
|
2455
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
2456
|
+
FROM vec_chat_messages v
|
|
2457
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
2458
|
+
WHERE v.embedding MATCH ?
|
|
2459
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
2460
|
+
}
|
|
2388
2461
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
2389
2462
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
2390
2463
|
}
|
|
@@ -886,6 +886,17 @@ var MIGRATIONS = [
|
|
|
886
886
|
ON chat_messages(session_id, transcript_index)
|
|
887
887
|
WHERE transcript_index IS NOT NULL;
|
|
888
888
|
`
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
version: 18,
|
|
892
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
893
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
894
|
+
sql: `
|
|
895
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
896
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
897
|
+
embedding FLOAT[384]
|
|
898
|
+
);
|
|
899
|
+
`
|
|
889
900
|
}
|
|
890
901
|
];
|
|
891
902
|
function isVecExtensionLoaded(db) {
|
|
@@ -959,6 +970,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
959
970
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
960
971
|
version = Math.max(version, 17);
|
|
961
972
|
}
|
|
973
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
974
|
+
version = Math.max(version, 18);
|
|
975
|
+
}
|
|
962
976
|
return version;
|
|
963
977
|
}
|
|
964
978
|
function runMigrations(db) {
|
|
@@ -1075,6 +1089,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1075
1089
|
db.exec("PRAGMA user_version = 17");
|
|
1076
1090
|
}
|
|
1077
1091
|
}
|
|
1092
|
+
function ensureChatVectorTable(db) {
|
|
1093
|
+
if (!isVecExtensionLoaded(db))
|
|
1094
|
+
return;
|
|
1095
|
+
db.exec(`
|
|
1096
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1097
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1098
|
+
embedding FLOAT[384]
|
|
1099
|
+
);
|
|
1100
|
+
`);
|
|
1101
|
+
const current = getSchemaVersion(db);
|
|
1102
|
+
if (current < 18) {
|
|
1103
|
+
db.exec("PRAGMA user_version = 18");
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1078
1106
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1079
1107
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1080
1108
|
const current = getSchemaVersion(db);
|
|
@@ -1288,6 +1316,7 @@ class MemDatabase {
|
|
|
1288
1316
|
ensureObservationTypes(this.db);
|
|
1289
1317
|
ensureSessionSummaryColumns(this.db);
|
|
1290
1318
|
ensureChatMessageColumns(this.db);
|
|
1319
|
+
ensureChatVectorTable(this.db);
|
|
1291
1320
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1292
1321
|
}
|
|
1293
1322
|
loadVecExtension() {
|
|
@@ -1654,6 +1683,14 @@ class MemDatabase {
|
|
|
1654
1683
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1655
1684
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1656
1685
|
}
|
|
1686
|
+
getChatMessagesByIds(ids) {
|
|
1687
|
+
if (ids.length === 0)
|
|
1688
|
+
return [];
|
|
1689
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1690
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1691
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1692
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1693
|
+
}
|
|
1657
1694
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1658
1695
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1659
1696
|
WHERE session_id = ?
|
|
@@ -1730,6 +1767,39 @@ class MemDatabase {
|
|
|
1730
1767
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1731
1768
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1732
1769
|
}
|
|
1770
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1771
|
+
if (!this.vecAvailable)
|
|
1772
|
+
return;
|
|
1773
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1774
|
+
}
|
|
1775
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1776
|
+
if (!this.vecAvailable)
|
|
1777
|
+
return [];
|
|
1778
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1779
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1780
|
+
const transcriptPreference = `
|
|
1781
|
+
AND (
|
|
1782
|
+
c.source_kind = 'transcript'
|
|
1783
|
+
OR NOT EXISTS (
|
|
1784
|
+
SELECT 1 FROM chat_messages t2
|
|
1785
|
+
WHERE t2.session_id = c.session_id
|
|
1786
|
+
AND t2.source_kind = 'transcript'
|
|
1787
|
+
)
|
|
1788
|
+
)`;
|
|
1789
|
+
if (projectId !== null) {
|
|
1790
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1791
|
+
FROM vec_chat_messages v
|
|
1792
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1793
|
+
WHERE v.embedding MATCH ?
|
|
1794
|
+
AND k = ?
|
|
1795
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1796
|
+
}
|
|
1797
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1798
|
+
FROM vec_chat_messages v
|
|
1799
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1800
|
+
WHERE v.embedding MATCH ?
|
|
1801
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1802
|
+
}
|
|
1733
1803
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1734
1804
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1735
1805
|
}
|
|
@@ -2514,6 +2584,9 @@ function composeEmbeddingText(obs) {
|
|
|
2514
2584
|
|
|
2515
2585
|
`);
|
|
2516
2586
|
}
|
|
2587
|
+
function composeChatEmbeddingText(text) {
|
|
2588
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2589
|
+
}
|
|
2517
2590
|
async function getPipeline() {
|
|
2518
2591
|
if (_pipeline)
|
|
2519
2592
|
return _pipeline;
|
|
@@ -3701,7 +3774,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
3701
3774
|
}
|
|
3702
3775
|
return messages;
|
|
3703
3776
|
}
|
|
3704
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3777
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3705
3778
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
3706
3779
|
...message,
|
|
3707
3780
|
text: message.text.trim()
|
|
@@ -3731,6 +3804,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
3731
3804
|
transcript_index: transcriptIndex
|
|
3732
3805
|
});
|
|
3733
3806
|
db.addToOutbox("chat_message", row.id);
|
|
3807
|
+
if (db.vecAvailable) {
|
|
3808
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
3809
|
+
if (embedding) {
|
|
3810
|
+
db.vecChatInsert(row.id, embedding);
|
|
3811
|
+
}
|
|
3812
|
+
}
|
|
3734
3813
|
imported++;
|
|
3735
3814
|
}
|
|
3736
3815
|
return { imported, total: messages.length };
|
|
@@ -4256,7 +4335,7 @@ async function main() {
|
|
|
4256
4335
|
try {
|
|
4257
4336
|
if (event.session_id) {
|
|
4258
4337
|
persistRawToolChronology(event, config.user_id, config.device_id);
|
|
4259
|
-
syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4338
|
+
await syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4260
4339
|
}
|
|
4261
4340
|
const textToScan = extractScanText(event);
|
|
4262
4341
|
if (textToScan) {
|
|
@@ -680,6 +680,17 @@ var MIGRATIONS = [
|
|
|
680
680
|
ON chat_messages(session_id, transcript_index)
|
|
681
681
|
WHERE transcript_index IS NOT NULL;
|
|
682
682
|
`
|
|
683
|
+
},
|
|
684
|
+
{
|
|
685
|
+
version: 18,
|
|
686
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
687
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
688
|
+
sql: `
|
|
689
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
690
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
691
|
+
embedding FLOAT[384]
|
|
692
|
+
);
|
|
693
|
+
`
|
|
683
694
|
}
|
|
684
695
|
];
|
|
685
696
|
function isVecExtensionLoaded(db) {
|
|
@@ -753,6 +764,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
753
764
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
754
765
|
version = Math.max(version, 17);
|
|
755
766
|
}
|
|
767
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
768
|
+
version = Math.max(version, 18);
|
|
769
|
+
}
|
|
756
770
|
return version;
|
|
757
771
|
}
|
|
758
772
|
function runMigrations(db) {
|
|
@@ -869,6 +883,20 @@ function ensureChatMessageColumns(db) {
|
|
|
869
883
|
db.exec("PRAGMA user_version = 17");
|
|
870
884
|
}
|
|
871
885
|
}
|
|
886
|
+
function ensureChatVectorTable(db) {
|
|
887
|
+
if (!isVecExtensionLoaded(db))
|
|
888
|
+
return;
|
|
889
|
+
db.exec(`
|
|
890
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
891
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
892
|
+
embedding FLOAT[384]
|
|
893
|
+
);
|
|
894
|
+
`);
|
|
895
|
+
const current = getSchemaVersion(db);
|
|
896
|
+
if (current < 18) {
|
|
897
|
+
db.exec("PRAGMA user_version = 18");
|
|
898
|
+
}
|
|
899
|
+
}
|
|
872
900
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
873
901
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
874
902
|
const current = getSchemaVersion(db);
|
|
@@ -1082,6 +1110,7 @@ class MemDatabase {
|
|
|
1082
1110
|
ensureObservationTypes(this.db);
|
|
1083
1111
|
ensureSessionSummaryColumns(this.db);
|
|
1084
1112
|
ensureChatMessageColumns(this.db);
|
|
1113
|
+
ensureChatVectorTable(this.db);
|
|
1085
1114
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1086
1115
|
}
|
|
1087
1116
|
loadVecExtension() {
|
|
@@ -1448,6 +1477,14 @@ class MemDatabase {
|
|
|
1448
1477
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1449
1478
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1450
1479
|
}
|
|
1480
|
+
getChatMessagesByIds(ids) {
|
|
1481
|
+
if (ids.length === 0)
|
|
1482
|
+
return [];
|
|
1483
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1484
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1485
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1486
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1487
|
+
}
|
|
1451
1488
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1452
1489
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1453
1490
|
WHERE session_id = ?
|
|
@@ -1524,6 +1561,39 @@ class MemDatabase {
|
|
|
1524
1561
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1525
1562
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1526
1563
|
}
|
|
1564
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1565
|
+
if (!this.vecAvailable)
|
|
1566
|
+
return;
|
|
1567
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1568
|
+
}
|
|
1569
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1570
|
+
if (!this.vecAvailable)
|
|
1571
|
+
return [];
|
|
1572
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1573
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1574
|
+
const transcriptPreference = `
|
|
1575
|
+
AND (
|
|
1576
|
+
c.source_kind = 'transcript'
|
|
1577
|
+
OR NOT EXISTS (
|
|
1578
|
+
SELECT 1 FROM chat_messages t2
|
|
1579
|
+
WHERE t2.session_id = c.session_id
|
|
1580
|
+
AND t2.source_kind = 'transcript'
|
|
1581
|
+
)
|
|
1582
|
+
)`;
|
|
1583
|
+
if (projectId !== null) {
|
|
1584
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1585
|
+
FROM vec_chat_messages v
|
|
1586
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1587
|
+
WHERE v.embedding MATCH ?
|
|
1588
|
+
AND k = ?
|
|
1589
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1590
|
+
}
|
|
1591
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1592
|
+
FROM vec_chat_messages v
|
|
1593
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1594
|
+
WHERE v.embedding MATCH ?
|
|
1595
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1596
|
+
}
|
|
1527
1597
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1528
1598
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1529
1599
|
}
|
|
@@ -2587,6 +2657,9 @@ function composeEmbeddingText(obs) {
|
|
|
2587
2657
|
|
|
2588
2658
|
`);
|
|
2589
2659
|
}
|
|
2660
|
+
function composeChatEmbeddingText(text) {
|
|
2661
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2662
|
+
}
|
|
2590
2663
|
async function getPipeline() {
|
|
2591
2664
|
if (_pipeline)
|
|
2592
2665
|
return _pipeline;
|
|
@@ -3954,7 +4027,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
3954
4027
|
}
|
|
3955
4028
|
return messages;
|
|
3956
4029
|
}
|
|
3957
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
4030
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3958
4031
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
3959
4032
|
...message,
|
|
3960
4033
|
text: message.text.trim()
|
|
@@ -3984,6 +4057,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
3984
4057
|
transcript_index: transcriptIndex
|
|
3985
4058
|
});
|
|
3986
4059
|
db.addToOutbox("chat_message", row.id);
|
|
4060
|
+
if (db.vecAvailable) {
|
|
4061
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
4062
|
+
if (embedding) {
|
|
4063
|
+
db.vecChatInsert(row.id, embedding);
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
3987
4066
|
imported++;
|
|
3988
4067
|
}
|
|
3989
4068
|
return { imported, total: messages.length };
|
|
@@ -4117,7 +4196,7 @@ async function main() {
|
|
|
4117
4196
|
try {
|
|
4118
4197
|
let importedChat = 0;
|
|
4119
4198
|
if (event.session_id) {
|
|
4120
|
-
const chatSync = syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4199
|
+
const chatSync = await syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
4121
4200
|
importedChat = chatSync.imported;
|
|
4122
4201
|
await upsertRollingHandoff(db, config, {
|
|
4123
4202
|
session_id: event.session_id,
|
package/dist/hooks/sentinel.js
CHANGED
|
@@ -756,6 +756,17 @@ var MIGRATIONS = [
|
|
|
756
756
|
ON chat_messages(session_id, transcript_index)
|
|
757
757
|
WHERE transcript_index IS NOT NULL;
|
|
758
758
|
`
|
|
759
|
+
},
|
|
760
|
+
{
|
|
761
|
+
version: 18,
|
|
762
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
763
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
764
|
+
sql: `
|
|
765
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
766
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
767
|
+
embedding FLOAT[384]
|
|
768
|
+
);
|
|
769
|
+
`
|
|
759
770
|
}
|
|
760
771
|
];
|
|
761
772
|
function isVecExtensionLoaded(db) {
|
|
@@ -829,6 +840,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
829
840
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
830
841
|
version = Math.max(version, 17);
|
|
831
842
|
}
|
|
843
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
844
|
+
version = Math.max(version, 18);
|
|
845
|
+
}
|
|
832
846
|
return version;
|
|
833
847
|
}
|
|
834
848
|
function runMigrations(db) {
|
|
@@ -945,6 +959,20 @@ function ensureChatMessageColumns(db) {
|
|
|
945
959
|
db.exec("PRAGMA user_version = 17");
|
|
946
960
|
}
|
|
947
961
|
}
|
|
962
|
+
function ensureChatVectorTable(db) {
|
|
963
|
+
if (!isVecExtensionLoaded(db))
|
|
964
|
+
return;
|
|
965
|
+
db.exec(`
|
|
966
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
967
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
968
|
+
embedding FLOAT[384]
|
|
969
|
+
);
|
|
970
|
+
`);
|
|
971
|
+
const current = getSchemaVersion(db);
|
|
972
|
+
if (current < 18) {
|
|
973
|
+
db.exec("PRAGMA user_version = 18");
|
|
974
|
+
}
|
|
975
|
+
}
|
|
948
976
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
949
977
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
950
978
|
const current = getSchemaVersion(db);
|
|
@@ -1158,6 +1186,7 @@ class MemDatabase {
|
|
|
1158
1186
|
ensureObservationTypes(this.db);
|
|
1159
1187
|
ensureSessionSummaryColumns(this.db);
|
|
1160
1188
|
ensureChatMessageColumns(this.db);
|
|
1189
|
+
ensureChatVectorTable(this.db);
|
|
1161
1190
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1162
1191
|
}
|
|
1163
1192
|
loadVecExtension() {
|
|
@@ -1524,6 +1553,14 @@ class MemDatabase {
|
|
|
1524
1553
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1525
1554
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1526
1555
|
}
|
|
1556
|
+
getChatMessagesByIds(ids) {
|
|
1557
|
+
if (ids.length === 0)
|
|
1558
|
+
return [];
|
|
1559
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1560
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1561
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1562
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1563
|
+
}
|
|
1527
1564
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1528
1565
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1529
1566
|
WHERE session_id = ?
|
|
@@ -1600,6 +1637,39 @@ class MemDatabase {
|
|
|
1600
1637
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1601
1638
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1602
1639
|
}
|
|
1640
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1641
|
+
if (!this.vecAvailable)
|
|
1642
|
+
return;
|
|
1643
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1644
|
+
}
|
|
1645
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1646
|
+
if (!this.vecAvailable)
|
|
1647
|
+
return [];
|
|
1648
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1649
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1650
|
+
const transcriptPreference = `
|
|
1651
|
+
AND (
|
|
1652
|
+
c.source_kind = 'transcript'
|
|
1653
|
+
OR NOT EXISTS (
|
|
1654
|
+
SELECT 1 FROM chat_messages t2
|
|
1655
|
+
WHERE t2.session_id = c.session_id
|
|
1656
|
+
AND t2.source_kind = 'transcript'
|
|
1657
|
+
)
|
|
1658
|
+
)`;
|
|
1659
|
+
if (projectId !== null) {
|
|
1660
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1661
|
+
FROM vec_chat_messages v
|
|
1662
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1663
|
+
WHERE v.embedding MATCH ?
|
|
1664
|
+
AND k = ?
|
|
1665
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1666
|
+
}
|
|
1667
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1668
|
+
FROM vec_chat_messages v
|
|
1669
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1670
|
+
WHERE v.embedding MATCH ?
|
|
1671
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1672
|
+
}
|
|
1603
1673
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1604
1674
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1605
1675
|
}
|