engrm 0.4.27 → 0.4.28

package/dist/server.js

@@ -14230,6 +14230,17 @@ var MIGRATIONS = [
         ON chat_messages(session_id, transcript_index)
         WHERE transcript_index IS NOT NULL;
     `
+  },
+  {
+    version: 18,
+    description: "Add sqlite-vec semantic search for chat recall",
+    condition: (db) => isVecExtensionLoaded(db),
+    sql: `
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+        chat_message_id INTEGER PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -14303,6 +14314,9 @@ function inferLegacySchemaVersion(db) {
   if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
     version2 = Math.max(version2, 17);
   }
+  if (tableExists(db, "vec_chat_messages")) {
+    version2 = Math.max(version2, 18);
+  }
   return version2;
 }
 function runMigrations(db) {
@@ -14419,6 +14433,20 @@ function ensureChatMessageColumns(db) {
     db.exec("PRAGMA user_version = 17");
   }
 }
+function ensureChatVectorTable(db) {
+  if (!isVecExtensionLoaded(db))
+    return;
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+      chat_message_id INTEGER PRIMARY KEY,
+      embedding FLOAT[384]
+    );
+  `);
+  const current = getSchemaVersion(db);
+  if (current < 18) {
+    db.exec("PRAGMA user_version = 18");
+  }
+}
 function ensureSyncOutboxSupportsChatMessages(db) {
   if (syncOutboxSupportsChatMessages(db)) {
     const current = getSchemaVersion(db);
@@ -14632,6 +14660,7 @@ class MemDatabase {
     ensureObservationTypes(this.db);
     ensureSessionSummaryColumns(this.db);
     ensureChatMessageColumns(this.db);
+    ensureChatVectorTable(this.db);
     ensureSyncOutboxSupportsChatMessages(this.db);
   }
   loadVecExtension() {
@@ -14998,6 +15027,14 @@ class MemDatabase {
   getChatMessageByRemoteSourceId(remoteSourceId) {
     return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
   }
+  getChatMessagesByIds(ids) {
+    if (ids.length === 0)
+      return [];
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
+    const order = new Map(ids.map((id, index) => [id, index]));
+    return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
+  }
   getSessionChatMessages(sessionId, limit = 50) {
     return this.db.query(`SELECT * FROM chat_messages
          WHERE session_id = ?
@@ -15074,6 +15111,39 @@ class MemDatabase {
          ORDER BY created_at_epoch DESC, id DESC
          LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
   }
+  vecChatInsert(chatMessageId, embedding) {
+    if (!this.vecAvailable)
+      return;
+    this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
+  }
+  searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
+    if (!this.vecAvailable)
+      return [];
+    const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
+    const visibilityClause = userId ? " AND c.user_id = ?" : "";
+    const transcriptPreference = `
+      AND (
+        c.source_kind = 'transcript'
+        OR NOT EXISTS (
+          SELECT 1 FROM chat_messages t2
+          WHERE t2.session_id = c.session_id
+            AND t2.source_kind = 'transcript'
+        )
+      )`;
+    if (projectId !== null) {
+      return this.db.query(`SELECT v.chat_message_id, v.distance
+           FROM vec_chat_messages v
+           JOIN chat_messages c ON c.id = v.chat_message_id
+           WHERE v.embedding MATCH ?
+             AND k = ?
+             AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
+    }
+    return this.db.query(`SELECT v.chat_message_id, v.distance
+         FROM vec_chat_messages v
+         JOIN chat_messages c ON c.id = v.chat_message_id
+         WHERE v.embedding MATCH ?
+           AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
+  }
   getTranscriptChatMessage(sessionId, transcriptIndex) {
     return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
   }
@@ -15815,6 +15885,9 @@ function composeEmbeddingText(obs) {
 
 `);
 }
+function composeChatEmbeddingText(text) {
+  return text.replace(/\s+/g, " ").trim().slice(0, 2000);
+}
 async function getPipeline() {
   if (_pipeline)
     return _pipeline;
@@ -16322,7 +16395,7 @@ function sanitizeFtsQuery(query) {
 }
 
 // src/tools/search-chat.ts
-function searchChat(db, input) {
+async function searchChat(db, input) {
   const limit = Math.max(1, Math.min(input.limit ?? 20, 100));
   const projectScoped = input.project_scoped !== false;
   let projectId = null;
@@ -16336,15 +16409,39 @@ function searchChat(db, input) {
       projectName = project.name;
     }
   }
-  const messages = db.searchChatMessages(input.query, projectId, limit, input.user_id);
+  const lexical = db.searchChatMessages(input.query, projectId, limit * 2, input.user_id);
+  let semantic = [];
+  const queryEmbedding = db.vecAvailable ? await embedText(composeChatEmbeddingText(queryForEmbedding(input.query))) : null;
+  if (queryEmbedding && db.vecAvailable) {
+    semantic = db.searchChatVec(queryEmbedding, projectId, limit * 2, input.user_id);
+  }
+  const messageIds = mergeChatResults(lexical, semantic, limit);
+  const messages = messageIds.length > 0 ? db.getChatMessagesByIds(messageIds) : [];
   return {
     messages,
     project: projectName,
     session_count: countDistinctSessions(messages),
     source_summary: summarizeChatSources(messages),
-    transcript_backed: messages.some((message) => message.source_kind === "transcript")
+    transcript_backed: messages.some((message) => message.source_kind === "transcript"),
+    semantic_backed: semantic.length > 0
   };
 }
+var RRF_K2 = 40;
+function mergeChatResults(lexical, semantic, limit) {
+  const scores = new Map;
+  for (let rank = 0;rank < lexical.length; rank++) {
+    const message = lexical[rank];
+    scores.set(message.id, (scores.get(message.id) ?? 0) + 1 / (RRF_K2 + rank + 1));
+  }
+  for (let rank = 0;rank < semantic.length; rank++) {
+    const match = semantic[rank];
+    scores.set(match.chat_message_id, (scores.get(match.chat_message_id) ?? 0) + 1 / (RRF_K2 + rank + 1));
+  }
+  return Array.from(scores.entries()).sort((a, b) => b[1] - a[1]).slice(0, limit).map(([id]) => id);
+}
+function queryForEmbedding(query) {
+  return query.replace(/\s+/g, " ").trim().slice(0, 400);
+}
 function summarizeChatSources(messages) {
   return messages.reduce((summary, message) => {
     summary[message.source_kind] += 1;
@@ -16368,13 +16465,13 @@ async function searchRecall(db, input) {
   const limit = Math.max(1, Math.min(input.limit ?? 10, 50));
   const [memory, chat] = await Promise.all([
     searchObservations(db, input),
-    Promise.resolve(searchChat(db, {
+    searchChat(db, {
       query,
       limit: limit * 2,
       project_scoped: input.project_scoped,
       cwd: input.cwd,
       user_id: input.user_id
-    }))
+    })
   ]);
   const merged = mergeRecallResults(memory.observations, chat.messages, limit);
   return {
@@ -18220,6 +18317,9 @@ function buildSuggestedTools(sessions, requestCount, toolCount, observationCount
   if (requestCount > 0 || toolCount > 0) {
     suggested.push("activity_feed");
   }
+  if (requestCount > 0 || recentChatCount > 0 || observationCount > 0) {
+    suggested.push("search_recall");
+  }
   if (observationCount > 0) {
     suggested.push("tool_memory_index", "capture_git_worktree");
   }
@@ -18317,6 +18417,8 @@ function buildFallbackSuggestedTools(sessionCount, requestCount, toolCount, obse
     suggested.push("recent_sessions");
   if (requestCount > 0 || toolCount > 0)
     suggested.push("activity_feed");
+  if (requestCount > 0 || chatCount > 0 || observationCount > 0)
+    suggested.push("search_recall");
   if (observationCount > 0)
     suggested.push("tool_memory_index", "capture_git_worktree");
   if (sessionCount > 0)
@@ -19192,6 +19294,9 @@ function buildSuggestedTools2(context, transcriptBackedChat) {
   if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentToolEvents?.length ?? 0) > 0) {
     tools.push("activity_feed");
   }
+  if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentChatMessages?.length ?? 0) > 0 || context.observations.length > 0) {
+    tools.push("search_recall");
+  }
   if (context.observations.length > 0) {
     tools.push("tool_memory_index", "capture_git_worktree");
   }
@@ -21177,7 +21282,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
   }
   return messages;
 }
-function syncTranscriptChat(db, config2, sessionId, cwd, transcriptPath) {
+async function syncTranscriptChat(db, config2, sessionId, cwd, transcriptPath) {
   const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
     ...message,
     text: message.text.trim()
@@ -21207,6 +21312,12 @@ function syncTranscriptChat(db, config2, sessionId, cwd, transcriptPath) {
       transcript_index: transcriptIndex
     });
     db.addToOutbox("chat_message", row.id);
+    if (db.vecAvailable) {
+      const embedding = await embedText(composeChatEmbeddingText(message.text));
+      if (embedding) {
+        db.vecChatInsert(row.id, embedding);
+      }
+    }
     imported++;
   }
   return { imported, total: messages.length };
@@ -21282,7 +21393,7 @@ process.on("SIGTERM", () => {
 });
 var server = new McpServer({
   name: "engrm",
-  version: "0.4.27"
+  version: "0.4.28"
 });
 server.tool("save_observation", "Save an observation to memory", {
   type: exports_external.enum([
@@ -22506,7 +22617,7 @@ server.tool("refresh_chat_recall", "Hydrate the separate chat lane from the curr
       content: [{ type: "text", text: "No session available to hydrate chat recall from." }]
     };
   }
-  const result = syncTranscriptChat(db, config2, sessionId, cwd, params.transcript_path);
+  const result = await syncTranscriptChat(db, config2, sessionId, cwd, params.transcript_path);
   return {
     content: [
       {
@@ -22622,10 +22733,10 @@ server.tool("search_chat", "Search the separate chat lane without mixing it into
   cwd: exports_external.string().optional(),
   user_id: exports_external.string().optional()
 }, async (params) => {
-  const result = searchChat(db, params);
+  const result = await searchChat(db, params);
   const projectLine = result.project ? `Project: ${result.project}
 ` : "";
-  const coverageLine = `Coverage: ${result.messages.length} matches across ${result.session_count} session${result.session_count === 1 ? "" : "s"} ` + `· transcript ${result.source_summary.transcript} · hook ${result.source_summary.hook}
+  const coverageLine = `Coverage: ${result.messages.length} matches across ${result.session_count} session${result.session_count === 1 ? "" : "s"} ` + `· transcript ${result.source_summary.transcript} · hook ${result.source_summary.hook}` + `${result.semantic_backed ? " · semantic yes" : ""}
 ` + `${result.transcript_backed ? "" : `Hint: run refresh_chat_recall if this looks under-captured.
 `}`;
   const rows = result.messages.length > 0 ? result.messages.map((msg) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "engrm",
-  "version": "0.4.27",
+  "version": "0.4.28",
   "description": "Shared memory across devices, sessions, and coding agents",
   "mcpName": "io.github.dr12hes/engrm",
   "type": "module",