engrm 0.4.27 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/cli.js +84 -0
- package/dist/hooks/elicitation-result.js +73 -0
- package/dist/hooks/post-tool-use.js +81 -2
- package/dist/hooks/pre-compact.js +81 -2
- package/dist/hooks/sentinel.js +70 -0
- package/dist/hooks/session-start.js +77 -1
- package/dist/hooks/stop.js +88 -3
- package/dist/hooks/user-prompt-submit.js +146 -61
- package/dist/server.js +121 -10
- package/package.json +1 -1
|
@@ -927,6 +927,9 @@ function composeEmbeddingText(obs) {
|
|
|
927
927
|
|
|
928
928
|
`);
|
|
929
929
|
}
|
|
930
|
+
function composeChatEmbeddingText(text) {
|
|
931
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
932
|
+
}
|
|
930
933
|
async function getPipeline() {
|
|
931
934
|
if (_pipeline)
|
|
932
935
|
return _pipeline;
|
|
@@ -3057,7 +3060,7 @@ import { existsSync as existsSync3, readFileSync as readFileSync2, writeFileSync
|
|
|
3057
3060
|
import { join as join3 } from "node:path";
|
|
3058
3061
|
import { homedir } from "node:os";
|
|
3059
3062
|
var STATE_PATH = join3(homedir(), ".engrm", "config-fingerprint.json");
|
|
3060
|
-
var CLIENT_VERSION = "0.4.
|
|
3063
|
+
var CLIENT_VERSION = "0.4.28";
|
|
3061
3064
|
function hashFile(filePath) {
|
|
3062
3065
|
try {
|
|
3063
3066
|
if (!existsSync3(filePath))
|
|
@@ -4252,6 +4255,17 @@ var MIGRATIONS = [
|
|
|
4252
4255
|
ON chat_messages(session_id, transcript_index)
|
|
4253
4256
|
WHERE transcript_index IS NOT NULL;
|
|
4254
4257
|
`
|
|
4258
|
+
},
|
|
4259
|
+
{
|
|
4260
|
+
version: 18,
|
|
4261
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
4262
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
4263
|
+
sql: `
|
|
4264
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
4265
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
4266
|
+
embedding FLOAT[384]
|
|
4267
|
+
);
|
|
4268
|
+
`
|
|
4255
4269
|
}
|
|
4256
4270
|
];
|
|
4257
4271
|
function isVecExtensionLoaded(db) {
|
|
@@ -4325,6 +4339,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
4325
4339
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
4326
4340
|
version = Math.max(version, 17);
|
|
4327
4341
|
}
|
|
4342
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
4343
|
+
version = Math.max(version, 18);
|
|
4344
|
+
}
|
|
4328
4345
|
return version;
|
|
4329
4346
|
}
|
|
4330
4347
|
function runMigrations(db) {
|
|
@@ -4441,6 +4458,20 @@ function ensureChatMessageColumns(db) {
|
|
|
4441
4458
|
db.exec("PRAGMA user_version = 17");
|
|
4442
4459
|
}
|
|
4443
4460
|
}
|
|
4461
|
+
function ensureChatVectorTable(db) {
|
|
4462
|
+
if (!isVecExtensionLoaded(db))
|
|
4463
|
+
return;
|
|
4464
|
+
db.exec(`
|
|
4465
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
4466
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
4467
|
+
embedding FLOAT[384]
|
|
4468
|
+
);
|
|
4469
|
+
`);
|
|
4470
|
+
const current = getSchemaVersion(db);
|
|
4471
|
+
if (current < 18) {
|
|
4472
|
+
db.exec("PRAGMA user_version = 18");
|
|
4473
|
+
}
|
|
4474
|
+
}
|
|
4444
4475
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
4445
4476
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
4446
4477
|
const current = getSchemaVersion(db);
|
|
@@ -4574,6 +4605,7 @@ class MemDatabase {
|
|
|
4574
4605
|
ensureObservationTypes(this.db);
|
|
4575
4606
|
ensureSessionSummaryColumns(this.db);
|
|
4576
4607
|
ensureChatMessageColumns(this.db);
|
|
4608
|
+
ensureChatVectorTable(this.db);
|
|
4577
4609
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
4578
4610
|
}
|
|
4579
4611
|
loadVecExtension() {
|
|
@@ -4940,6 +4972,14 @@ class MemDatabase {
|
|
|
4940
4972
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
4941
4973
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
4942
4974
|
}
|
|
4975
|
+
getChatMessagesByIds(ids) {
|
|
4976
|
+
if (ids.length === 0)
|
|
4977
|
+
return [];
|
|
4978
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
4979
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
4980
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
4981
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
4982
|
+
}
|
|
4943
4983
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
4944
4984
|
return this.db.query(`SELECT * FROM chat_messages
|
|
4945
4985
|
WHERE session_id = ?
|
|
@@ -5016,6 +5056,39 @@ class MemDatabase {
|
|
|
5016
5056
|
ORDER BY created_at_epoch DESC, id DESC
|
|
5017
5057
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
5018
5058
|
}
|
|
5059
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
5060
|
+
if (!this.vecAvailable)
|
|
5061
|
+
return;
|
|
5062
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
5063
|
+
}
|
|
5064
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
5065
|
+
if (!this.vecAvailable)
|
|
5066
|
+
return [];
|
|
5067
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
5068
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
5069
|
+
const transcriptPreference = `
|
|
5070
|
+
AND (
|
|
5071
|
+
c.source_kind = 'transcript'
|
|
5072
|
+
OR NOT EXISTS (
|
|
5073
|
+
SELECT 1 FROM chat_messages t2
|
|
5074
|
+
WHERE t2.session_id = c.session_id
|
|
5075
|
+
AND t2.source_kind = 'transcript'
|
|
5076
|
+
)
|
|
5077
|
+
)`;
|
|
5078
|
+
if (projectId !== null) {
|
|
5079
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
5080
|
+
FROM vec_chat_messages v
|
|
5081
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
5082
|
+
WHERE v.embedding MATCH ?
|
|
5083
|
+
AND k = ?
|
|
5084
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
5085
|
+
}
|
|
5086
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
5087
|
+
FROM vec_chat_messages v
|
|
5088
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
5089
|
+
WHERE v.embedding MATCH ?
|
|
5090
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
5091
|
+
}
|
|
5019
5092
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
5020
5093
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
5021
5094
|
}
|
|
@@ -5669,6 +5742,9 @@ function formatInspectHints(context, visibleObservationIds = []) {
|
|
|
5669
5742
|
if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentToolEvents?.length ?? 0) > 0 || (context.recentChatMessages?.length ?? 0) > 0) {
|
|
5670
5743
|
hints.push("activity_feed");
|
|
5671
5744
|
}
|
|
5745
|
+
if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentChatMessages?.length ?? 0) > 0 || context.observations.length > 0) {
|
|
5746
|
+
hints.push("search_recall");
|
|
5747
|
+
}
|
|
5672
5748
|
if (context.observations.length > 0) {
|
|
5673
5749
|
hints.push("memory_console");
|
|
5674
5750
|
}
|
package/dist/hooks/stop.js
CHANGED
|
@@ -993,6 +993,17 @@ var MIGRATIONS = [
|
|
|
993
993
|
ON chat_messages(session_id, transcript_index)
|
|
994
994
|
WHERE transcript_index IS NOT NULL;
|
|
995
995
|
`
|
|
996
|
+
},
|
|
997
|
+
{
|
|
998
|
+
version: 18,
|
|
999
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
1000
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
1001
|
+
sql: `
|
|
1002
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1003
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1004
|
+
embedding FLOAT[384]
|
|
1005
|
+
);
|
|
1006
|
+
`
|
|
996
1007
|
}
|
|
997
1008
|
];
|
|
998
1009
|
function isVecExtensionLoaded(db) {
|
|
@@ -1066,6 +1077,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
1066
1077
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
1067
1078
|
version = Math.max(version, 17);
|
|
1068
1079
|
}
|
|
1080
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
1081
|
+
version = Math.max(version, 18);
|
|
1082
|
+
}
|
|
1069
1083
|
return version;
|
|
1070
1084
|
}
|
|
1071
1085
|
function runMigrations(db) {
|
|
@@ -1182,6 +1196,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1182
1196
|
db.exec("PRAGMA user_version = 17");
|
|
1183
1197
|
}
|
|
1184
1198
|
}
|
|
1199
|
+
function ensureChatVectorTable(db) {
|
|
1200
|
+
if (!isVecExtensionLoaded(db))
|
|
1201
|
+
return;
|
|
1202
|
+
db.exec(`
|
|
1203
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1204
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1205
|
+
embedding FLOAT[384]
|
|
1206
|
+
);
|
|
1207
|
+
`);
|
|
1208
|
+
const current = getSchemaVersion(db);
|
|
1209
|
+
if (current < 18) {
|
|
1210
|
+
db.exec("PRAGMA user_version = 18");
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1185
1213
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1186
1214
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1187
1215
|
const current = getSchemaVersion(db);
|
|
@@ -1315,6 +1343,7 @@ class MemDatabase {
|
|
|
1315
1343
|
ensureObservationTypes(this.db);
|
|
1316
1344
|
ensureSessionSummaryColumns(this.db);
|
|
1317
1345
|
ensureChatMessageColumns(this.db);
|
|
1346
|
+
ensureChatVectorTable(this.db);
|
|
1318
1347
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1319
1348
|
}
|
|
1320
1349
|
loadVecExtension() {
|
|
@@ -1681,6 +1710,14 @@ class MemDatabase {
|
|
|
1681
1710
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1682
1711
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1683
1712
|
}
|
|
1713
|
+
getChatMessagesByIds(ids) {
|
|
1714
|
+
if (ids.length === 0)
|
|
1715
|
+
return [];
|
|
1716
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1717
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1718
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1719
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1720
|
+
}
|
|
1684
1721
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1685
1722
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1686
1723
|
WHERE session_id = ?
|
|
@@ -1757,6 +1794,39 @@ class MemDatabase {
|
|
|
1757
1794
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1758
1795
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1759
1796
|
}
|
|
1797
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1798
|
+
if (!this.vecAvailable)
|
|
1799
|
+
return;
|
|
1800
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1801
|
+
}
|
|
1802
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1803
|
+
if (!this.vecAvailable)
|
|
1804
|
+
return [];
|
|
1805
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1806
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1807
|
+
const transcriptPreference = `
|
|
1808
|
+
AND (
|
|
1809
|
+
c.source_kind = 'transcript'
|
|
1810
|
+
OR NOT EXISTS (
|
|
1811
|
+
SELECT 1 FROM chat_messages t2
|
|
1812
|
+
WHERE t2.session_id = c.session_id
|
|
1813
|
+
AND t2.source_kind = 'transcript'
|
|
1814
|
+
)
|
|
1815
|
+
)`;
|
|
1816
|
+
if (projectId !== null) {
|
|
1817
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1818
|
+
FROM vec_chat_messages v
|
|
1819
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1820
|
+
WHERE v.embedding MATCH ?
|
|
1821
|
+
AND k = ?
|
|
1822
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1823
|
+
}
|
|
1824
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1825
|
+
FROM vec_chat_messages v
|
|
1826
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1827
|
+
WHERE v.embedding MATCH ?
|
|
1828
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1829
|
+
}
|
|
1760
1830
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1761
1831
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1762
1832
|
}
|
|
@@ -2719,6 +2789,9 @@ function composeEmbeddingText(obs) {
|
|
|
2719
2789
|
|
|
2720
2790
|
`);
|
|
2721
2791
|
}
|
|
2792
|
+
function composeChatEmbeddingText(text) {
|
|
2793
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2794
|
+
}
|
|
2722
2795
|
async function getPipeline() {
|
|
2723
2796
|
if (_pipeline)
|
|
2724
2797
|
return _pipeline;
|
|
@@ -3009,7 +3082,7 @@ function buildBeacon(db, config, sessionId, metrics) {
|
|
|
3009
3082
|
sentinel_used: valueSignals.security_findings_count > 0,
|
|
3010
3083
|
risk_score: riskScore,
|
|
3011
3084
|
stacks_detected: stacks,
|
|
3012
|
-
client_version: "0.4.
|
|
3085
|
+
client_version: "0.4.28",
|
|
3013
3086
|
context_observations_injected: metrics?.contextObsInjected ?? 0,
|
|
3014
3087
|
context_total_available: metrics?.contextTotalAvailable ?? 0,
|
|
3015
3088
|
recall_attempts: metrics?.recallAttempts ?? 0,
|
|
@@ -3910,7 +3983,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
3910
3983
|
}
|
|
3911
3984
|
return messages;
|
|
3912
3985
|
}
|
|
3913
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3986
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
3914
3987
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
3915
3988
|
...message,
|
|
3916
3989
|
text: message.text.trim()
|
|
@@ -3940,6 +4013,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
3940
4013
|
transcript_index: transcriptIndex
|
|
3941
4014
|
});
|
|
3942
4015
|
db.addToOutbox("chat_message", row.id);
|
|
4016
|
+
if (db.vecAvailable) {
|
|
4017
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
4018
|
+
if (embedding) {
|
|
4019
|
+
db.vecChatInsert(row.id, embedding);
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
3943
4022
|
imported++;
|
|
3944
4023
|
}
|
|
3945
4024
|
return { imported, total: messages.length };
|
|
@@ -4485,7 +4564,7 @@ async function main() {
|
|
|
4485
4564
|
try {
|
|
4486
4565
|
if (event.session_id) {
|
|
4487
4566
|
db.completeSession(event.session_id);
|
|
4488
|
-
syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path);
|
|
4567
|
+
await syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path);
|
|
4489
4568
|
if (event.last_assistant_message) {
|
|
4490
4569
|
try {
|
|
4491
4570
|
const detected = detectProject(event.cwd);
|
|
@@ -4508,6 +4587,12 @@ async function main() {
|
|
|
4508
4587
|
source_kind: "hook"
|
|
4509
4588
|
});
|
|
4510
4589
|
db.addToOutbox("chat_message", chatMessage.id);
|
|
4590
|
+
if (db.vecAvailable) {
|
|
4591
|
+
const chatEmbedding = await embedText(composeChatEmbeddingText(event.last_assistant_message));
|
|
4592
|
+
if (chatEmbedding) {
|
|
4593
|
+
db.vecChatInsert(chatMessage.id, chatEmbedding);
|
|
4594
|
+
}
|
|
4595
|
+
}
|
|
4511
4596
|
}
|
|
4512
4597
|
createAssistantCheckpoint(db, event.session_id, event.cwd, event.last_assistant_message);
|
|
4513
4598
|
} catch {}
|
|
@@ -824,6 +824,17 @@ var MIGRATIONS = [
|
|
|
824
824
|
ON chat_messages(session_id, transcript_index)
|
|
825
825
|
WHERE transcript_index IS NOT NULL;
|
|
826
826
|
`
|
|
827
|
+
},
|
|
828
|
+
{
|
|
829
|
+
version: 18,
|
|
830
|
+
description: "Add sqlite-vec semantic search for chat recall",
|
|
831
|
+
condition: (db) => isVecExtensionLoaded(db),
|
|
832
|
+
sql: `
|
|
833
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
834
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
835
|
+
embedding FLOAT[384]
|
|
836
|
+
);
|
|
837
|
+
`
|
|
827
838
|
}
|
|
828
839
|
];
|
|
829
840
|
function isVecExtensionLoaded(db) {
|
|
@@ -897,6 +908,9 @@ function inferLegacySchemaVersion(db) {
|
|
|
897
908
|
if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
|
|
898
909
|
version = Math.max(version, 17);
|
|
899
910
|
}
|
|
911
|
+
if (tableExists(db, "vec_chat_messages")) {
|
|
912
|
+
version = Math.max(version, 18);
|
|
913
|
+
}
|
|
900
914
|
return version;
|
|
901
915
|
}
|
|
902
916
|
function runMigrations(db) {
|
|
@@ -1013,6 +1027,20 @@ function ensureChatMessageColumns(db) {
|
|
|
1013
1027
|
db.exec("PRAGMA user_version = 17");
|
|
1014
1028
|
}
|
|
1015
1029
|
}
|
|
1030
|
+
function ensureChatVectorTable(db) {
|
|
1031
|
+
if (!isVecExtensionLoaded(db))
|
|
1032
|
+
return;
|
|
1033
|
+
db.exec(`
|
|
1034
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
|
|
1035
|
+
chat_message_id INTEGER PRIMARY KEY,
|
|
1036
|
+
embedding FLOAT[384]
|
|
1037
|
+
);
|
|
1038
|
+
`);
|
|
1039
|
+
const current = getSchemaVersion(db);
|
|
1040
|
+
if (current < 18) {
|
|
1041
|
+
db.exec("PRAGMA user_version = 18");
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1016
1044
|
function ensureSyncOutboxSupportsChatMessages(db) {
|
|
1017
1045
|
if (syncOutboxSupportsChatMessages(db)) {
|
|
1018
1046
|
const current = getSchemaVersion(db);
|
|
@@ -1226,6 +1254,7 @@ class MemDatabase {
|
|
|
1226
1254
|
ensureObservationTypes(this.db);
|
|
1227
1255
|
ensureSessionSummaryColumns(this.db);
|
|
1228
1256
|
ensureChatMessageColumns(this.db);
|
|
1257
|
+
ensureChatVectorTable(this.db);
|
|
1229
1258
|
ensureSyncOutboxSupportsChatMessages(this.db);
|
|
1230
1259
|
}
|
|
1231
1260
|
loadVecExtension() {
|
|
@@ -1592,6 +1621,14 @@ class MemDatabase {
|
|
|
1592
1621
|
getChatMessageByRemoteSourceId(remoteSourceId) {
|
|
1593
1622
|
return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
|
|
1594
1623
|
}
|
|
1624
|
+
getChatMessagesByIds(ids) {
|
|
1625
|
+
if (ids.length === 0)
|
|
1626
|
+
return [];
|
|
1627
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1628
|
+
const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
|
|
1629
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
1630
|
+
return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
|
|
1631
|
+
}
|
|
1595
1632
|
getSessionChatMessages(sessionId, limit = 50) {
|
|
1596
1633
|
return this.db.query(`SELECT * FROM chat_messages
|
|
1597
1634
|
WHERE session_id = ?
|
|
@@ -1668,6 +1705,39 @@ class MemDatabase {
|
|
|
1668
1705
|
ORDER BY created_at_epoch DESC, id DESC
|
|
1669
1706
|
LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
|
|
1670
1707
|
}
|
|
1708
|
+
vecChatInsert(chatMessageId, embedding) {
|
|
1709
|
+
if (!this.vecAvailable)
|
|
1710
|
+
return;
|
|
1711
|
+
this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
|
|
1712
|
+
}
|
|
1713
|
+
searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
|
|
1714
|
+
if (!this.vecAvailable)
|
|
1715
|
+
return [];
|
|
1716
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1717
|
+
const visibilityClause = userId ? " AND c.user_id = ?" : "";
|
|
1718
|
+
const transcriptPreference = `
|
|
1719
|
+
AND (
|
|
1720
|
+
c.source_kind = 'transcript'
|
|
1721
|
+
OR NOT EXISTS (
|
|
1722
|
+
SELECT 1 FROM chat_messages t2
|
|
1723
|
+
WHERE t2.session_id = c.session_id
|
|
1724
|
+
AND t2.source_kind = 'transcript'
|
|
1725
|
+
)
|
|
1726
|
+
)`;
|
|
1727
|
+
if (projectId !== null) {
|
|
1728
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1729
|
+
FROM vec_chat_messages v
|
|
1730
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1731
|
+
WHERE v.embedding MATCH ?
|
|
1732
|
+
AND k = ?
|
|
1733
|
+
AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
|
|
1734
|
+
}
|
|
1735
|
+
return this.db.query(`SELECT v.chat_message_id, v.distance
|
|
1736
|
+
FROM vec_chat_messages v
|
|
1737
|
+
JOIN chat_messages c ON c.id = v.chat_message_id
|
|
1738
|
+
WHERE v.embedding MATCH ?
|
|
1739
|
+
AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
|
|
1740
|
+
}
|
|
1671
1741
|
getTranscriptChatMessage(sessionId, transcriptIndex) {
|
|
1672
1742
|
return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
|
|
1673
1743
|
}
|
|
@@ -2026,6 +2096,68 @@ import { readFileSync as readFileSync3, existsSync as existsSync3 } from "node:f
|
|
|
2026
2096
|
import { join as join3 } from "node:path";
|
|
2027
2097
|
import { homedir as homedir2 } from "node:os";
|
|
2028
2098
|
|
|
2099
|
+
// src/embeddings/embedder.ts
|
|
2100
|
+
var _available = null;
|
|
2101
|
+
var _pipeline = null;
|
|
2102
|
+
var MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
2103
|
+
async function embedText(text) {
|
|
2104
|
+
const pipe = await getPipeline();
|
|
2105
|
+
if (!pipe)
|
|
2106
|
+
return null;
|
|
2107
|
+
try {
|
|
2108
|
+
const output = await pipe(text, { pooling: "mean", normalize: true });
|
|
2109
|
+
return new Float32Array(output.data);
|
|
2110
|
+
} catch {
|
|
2111
|
+
return null;
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
function composeEmbeddingText(obs) {
|
|
2115
|
+
const parts = [obs.title];
|
|
2116
|
+
if (obs.narrative)
|
|
2117
|
+
parts.push(obs.narrative);
|
|
2118
|
+
if (obs.facts) {
|
|
2119
|
+
try {
|
|
2120
|
+
const facts = JSON.parse(obs.facts);
|
|
2121
|
+
if (Array.isArray(facts) && facts.length > 0) {
|
|
2122
|
+
parts.push(facts.map((f) => `- ${f}`).join(`
|
|
2123
|
+
`));
|
|
2124
|
+
}
|
|
2125
|
+
} catch {
|
|
2126
|
+
parts.push(obs.facts);
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
if (obs.concepts) {
|
|
2130
|
+
try {
|
|
2131
|
+
const concepts = JSON.parse(obs.concepts);
|
|
2132
|
+
if (Array.isArray(concepts) && concepts.length > 0) {
|
|
2133
|
+
parts.push(concepts.join(", "));
|
|
2134
|
+
}
|
|
2135
|
+
} catch {}
|
|
2136
|
+
}
|
|
2137
|
+
return parts.join(`
|
|
2138
|
+
|
|
2139
|
+
`);
|
|
2140
|
+
}
|
|
2141
|
+
function composeChatEmbeddingText(text) {
|
|
2142
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 2000);
|
|
2143
|
+
}
|
|
2144
|
+
async function getPipeline() {
|
|
2145
|
+
if (_pipeline)
|
|
2146
|
+
return _pipeline;
|
|
2147
|
+
if (_available === false)
|
|
2148
|
+
return null;
|
|
2149
|
+
try {
|
|
2150
|
+
const { pipeline } = await import("@xenova/transformers");
|
|
2151
|
+
_pipeline = await pipeline("feature-extraction", MODEL_NAME);
|
|
2152
|
+
_available = true;
|
|
2153
|
+
return _pipeline;
|
|
2154
|
+
} catch (err) {
|
|
2155
|
+
_available = false;
|
|
2156
|
+
console.error(`[engrm] Local embedding model unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
2157
|
+
return null;
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2029
2161
|
// src/tools/save.ts
|
|
2030
2162
|
import { relative, isAbsolute } from "node:path";
|
|
2031
2163
|
|
|
@@ -2348,65 +2480,6 @@ function looksMeaningful(value) {
|
|
|
2348
2480
|
return true;
|
|
2349
2481
|
}
|
|
2350
2482
|
|
|
2351
|
-
// src/embeddings/embedder.ts
|
|
2352
|
-
var _available = null;
|
|
2353
|
-
var _pipeline = null;
|
|
2354
|
-
var MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
2355
|
-
async function embedText(text) {
|
|
2356
|
-
const pipe = await getPipeline();
|
|
2357
|
-
if (!pipe)
|
|
2358
|
-
return null;
|
|
2359
|
-
try {
|
|
2360
|
-
const output = await pipe(text, { pooling: "mean", normalize: true });
|
|
2361
|
-
return new Float32Array(output.data);
|
|
2362
|
-
} catch {
|
|
2363
|
-
return null;
|
|
2364
|
-
}
|
|
2365
|
-
}
|
|
2366
|
-
function composeEmbeddingText(obs) {
|
|
2367
|
-
const parts = [obs.title];
|
|
2368
|
-
if (obs.narrative)
|
|
2369
|
-
parts.push(obs.narrative);
|
|
2370
|
-
if (obs.facts) {
|
|
2371
|
-
try {
|
|
2372
|
-
const facts = JSON.parse(obs.facts);
|
|
2373
|
-
if (Array.isArray(facts) && facts.length > 0) {
|
|
2374
|
-
parts.push(facts.map((f) => `- ${f}`).join(`
|
|
2375
|
-
`));
|
|
2376
|
-
}
|
|
2377
|
-
} catch {
|
|
2378
|
-
parts.push(obs.facts);
|
|
2379
|
-
}
|
|
2380
|
-
}
|
|
2381
|
-
if (obs.concepts) {
|
|
2382
|
-
try {
|
|
2383
|
-
const concepts = JSON.parse(obs.concepts);
|
|
2384
|
-
if (Array.isArray(concepts) && concepts.length > 0) {
|
|
2385
|
-
parts.push(concepts.join(", "));
|
|
2386
|
-
}
|
|
2387
|
-
} catch {}
|
|
2388
|
-
}
|
|
2389
|
-
return parts.join(`
|
|
2390
|
-
|
|
2391
|
-
`);
|
|
2392
|
-
}
|
|
2393
|
-
async function getPipeline() {
|
|
2394
|
-
if (_pipeline)
|
|
2395
|
-
return _pipeline;
|
|
2396
|
-
if (_available === false)
|
|
2397
|
-
return null;
|
|
2398
|
-
try {
|
|
2399
|
-
const { pipeline } = await import("@xenova/transformers");
|
|
2400
|
-
_pipeline = await pipeline("feature-extraction", MODEL_NAME);
|
|
2401
|
-
_available = true;
|
|
2402
|
-
return _pipeline;
|
|
2403
|
-
} catch (err) {
|
|
2404
|
-
_available = false;
|
|
2405
|
-
console.error(`[engrm] Local embedding model unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
2406
|
-
return null;
|
|
2407
|
-
}
|
|
2408
|
-
}
|
|
2409
|
-
|
|
2410
2483
|
// src/capture/recurrence.ts
|
|
2411
2484
|
var DISTANCE_THRESHOLD = 0.15;
|
|
2412
2485
|
async function detectRecurrence(db, config, observation) {
|
|
@@ -2794,7 +2867,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
|
|
|
2794
2867
|
}
|
|
2795
2868
|
return messages;
|
|
2796
2869
|
}
|
|
2797
|
-
function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
2870
|
+
async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
2798
2871
|
const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
|
|
2799
2872
|
...message,
|
|
2800
2873
|
text: message.text.trim()
|
|
@@ -2824,6 +2897,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
|
|
|
2824
2897
|
transcript_index: transcriptIndex
|
|
2825
2898
|
});
|
|
2826
2899
|
db.addToOutbox("chat_message", row.id);
|
|
2900
|
+
if (db.vecAvailable) {
|
|
2901
|
+
const embedding = await embedText(composeChatEmbeddingText(message.text));
|
|
2902
|
+
if (embedding) {
|
|
2903
|
+
db.vecChatInsert(row.id, embedding);
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2827
2906
|
imported++;
|
|
2828
2907
|
}
|
|
2829
2908
|
return { imported, total: messages.length };
|
|
@@ -3349,7 +3428,7 @@ async function main() {
|
|
|
3349
3428
|
device_id: config.device_id,
|
|
3350
3429
|
agent: "claude-code"
|
|
3351
3430
|
});
|
|
3352
|
-
syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
3431
|
+
await syncTranscriptChat(db, config, event.session_id, event.cwd);
|
|
3353
3432
|
const chatMessage = db.insertChatMessage({
|
|
3354
3433
|
session_id: event.session_id,
|
|
3355
3434
|
project_id: project.id,
|
|
@@ -3361,6 +3440,12 @@ async function main() {
|
|
|
3361
3440
|
source_kind: "hook"
|
|
3362
3441
|
});
|
|
3363
3442
|
db.addToOutbox("chat_message", chatMessage.id);
|
|
3443
|
+
if (db.vecAvailable) {
|
|
3444
|
+
const chatEmbedding = await embedText(composeChatEmbeddingText(event.prompt));
|
|
3445
|
+
if (chatEmbedding) {
|
|
3446
|
+
db.vecChatInsert(chatMessage.id, chatEmbedding);
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3364
3449
|
const compactPrompt = event.prompt.replace(/\s+/g, " ").trim();
|
|
3365
3450
|
if (compactPrompt.length >= 8) {
|
|
3366
3451
|
const sessionPrompts = db.getSessionUserPrompts(event.session_id, 20);
|