engrm 0.4.26 → 0.4.28

package/dist/hooks/sentinel.js

@@ -756,6 +756,17 @@ var MIGRATIONS = [
         ON chat_messages(session_id, transcript_index)
         WHERE transcript_index IS NOT NULL;
     `
+  },
+  {
+    version: 18,
+    description: "Add sqlite-vec semantic search for chat recall",
+    condition: (db) => isVecExtensionLoaded(db),
+    sql: `
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+        chat_message_id INTEGER PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -829,6 +840,9 @@ function inferLegacySchemaVersion(db) {
   if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
     version = Math.max(version, 17);
   }
+  if (tableExists(db, "vec_chat_messages")) {
+    version = Math.max(version, 18);
+  }
   return version;
 }
 function runMigrations(db) {
@@ -945,6 +959,20 @@ function ensureChatMessageColumns(db) {
     db.exec("PRAGMA user_version = 17");
   }
 }
+function ensureChatVectorTable(db) {
+  if (!isVecExtensionLoaded(db))
+    return;
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+      chat_message_id INTEGER PRIMARY KEY,
+      embedding FLOAT[384]
+    );
+  `);
+  const current = getSchemaVersion(db);
+  if (current < 18) {
+    db.exec("PRAGMA user_version = 18");
+  }
+}
 function ensureSyncOutboxSupportsChatMessages(db) {
   if (syncOutboxSupportsChatMessages(db)) {
     const current = getSchemaVersion(db);
@@ -1158,6 +1186,7 @@ class MemDatabase {
     ensureObservationTypes(this.db);
     ensureSessionSummaryColumns(this.db);
     ensureChatMessageColumns(this.db);
+    ensureChatVectorTable(this.db);
     ensureSyncOutboxSupportsChatMessages(this.db);
   }
   loadVecExtension() {
@@ -1524,6 +1553,14 @@ class MemDatabase {
   getChatMessageByRemoteSourceId(remoteSourceId) {
     return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
   }
+  getChatMessagesByIds(ids) {
+    if (ids.length === 0)
+      return [];
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
+    const order = new Map(ids.map((id, index) => [id, index]));
+    return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
+  }
   getSessionChatMessages(sessionId, limit = 50) {
     return this.db.query(`SELECT * FROM chat_messages
          WHERE session_id = ?
@@ -1600,6 +1637,39 @@ class MemDatabase {
          ORDER BY created_at_epoch DESC, id DESC
          LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
   }
+  vecChatInsert(chatMessageId, embedding) {
+    if (!this.vecAvailable)
+      return;
+    this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
+  }
+  searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
+    if (!this.vecAvailable)
+      return [];
+    const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
+    const visibilityClause = userId ? " AND c.user_id = ?" : "";
+    const transcriptPreference = `
+      AND (
+        c.source_kind = 'transcript'
+        OR NOT EXISTS (
+          SELECT 1 FROM chat_messages t2
+          WHERE t2.session_id = c.session_id
+            AND t2.source_kind = 'transcript'
+        )
+      )`;
+    if (projectId !== null) {
+      return this.db.query(`SELECT v.chat_message_id, v.distance
+           FROM vec_chat_messages v
+           JOIN chat_messages c ON c.id = v.chat_message_id
+           WHERE v.embedding MATCH ?
+             AND k = ?
+             AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
+    }
+    return this.db.query(`SELECT v.chat_message_id, v.distance
+         FROM vec_chat_messages v
+         JOIN chat_messages c ON c.id = v.chat_message_id
+         WHERE v.embedding MATCH ?
+           AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
+  }
   getTranscriptChatMessage(sessionId, transcriptIndex) {
     return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
   }

package/dist/hooks/session-start.js

@@ -494,6 +494,8 @@ function getSessionStory(db, input) {
     summary,
     prompts,
     chat_messages: chatMessages,
+    chat_source_summary: summarizeChatSources(chatMessages),
+    chat_coverage_state: chatMessages.some((message) => message.source_kind === "transcript") ? "transcript-backed" : chatMessages.length > 0 ? "hook-only" : "none",
     tool_events: toolEvents,
     observations,
     handoffs,
@@ -591,6 +593,12 @@ function collectProvenanceSummary(observations) {
   }
   return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => b.count - a.count || a.tool.localeCompare(b.tool)).slice(0, 6);
 }
+function summarizeChatSources(messages) {
+  return messages.reduce((summary, message) => {
+    summary[message.source_kind] += 1;
+    return summary;
+  }, { transcript: 0, hook: 0 });
+}
 
 // src/tools/save.ts
 import { relative, isAbsolute } from "node:path";
@@ -919,6 +927,9 @@ function composeEmbeddingText(obs) {
 
 `);
 }
+function composeChatEmbeddingText(text) {
+  return text.replace(/\s+/g, " ").trim().slice(0, 2000);
+}
 async function getPipeline() {
   if (_pipeline)
     return _pipeline;
@@ -3049,7 +3060,7 @@ import { existsSync as existsSync3, readFileSync as readFileSync2, writeFileSync
 import { join as join3 } from "node:path";
 import { homedir } from "node:os";
 var STATE_PATH = join3(homedir(), ".engrm", "config-fingerprint.json");
-var CLIENT_VERSION = "0.4.26";
+var CLIENT_VERSION = "0.4.28";
 function hashFile(filePath) {
   try {
     if (!existsSync3(filePath))
@@ -4244,6 +4255,17 @@ var MIGRATIONS = [
         ON chat_messages(session_id, transcript_index)
         WHERE transcript_index IS NOT NULL;
     `
+  },
+  {
+    version: 18,
+    description: "Add sqlite-vec semantic search for chat recall",
+    condition: (db) => isVecExtensionLoaded(db),
+    sql: `
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+        chat_message_id INTEGER PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -4317,6 +4339,9 @@ function inferLegacySchemaVersion(db) {
   if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
     version = Math.max(version, 17);
   }
+  if (tableExists(db, "vec_chat_messages")) {
+    version = Math.max(version, 18);
+  }
   return version;
 }
 function runMigrations(db) {
@@ -4433,6 +4458,20 @@ function ensureChatMessageColumns(db) {
     db.exec("PRAGMA user_version = 17");
   }
 }
+function ensureChatVectorTable(db) {
+  if (!isVecExtensionLoaded(db))
+    return;
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+      chat_message_id INTEGER PRIMARY KEY,
+      embedding FLOAT[384]
+    );
+  `);
+  const current = getSchemaVersion(db);
+  if (current < 18) {
+    db.exec("PRAGMA user_version = 18");
+  }
+}
 function ensureSyncOutboxSupportsChatMessages(db) {
   if (syncOutboxSupportsChatMessages(db)) {
     const current = getSchemaVersion(db);
@@ -4566,6 +4605,7 @@ class MemDatabase {
     ensureObservationTypes(this.db);
     ensureSessionSummaryColumns(this.db);
     ensureChatMessageColumns(this.db);
+    ensureChatVectorTable(this.db);
     ensureSyncOutboxSupportsChatMessages(this.db);
   }
   loadVecExtension() {
@@ -4932,6 +4972,14 @@ class MemDatabase {
   getChatMessageByRemoteSourceId(remoteSourceId) {
     return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
   }
+  getChatMessagesByIds(ids) {
+    if (ids.length === 0)
+      return [];
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
+    const order = new Map(ids.map((id, index) => [id, index]));
+    return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
+  }
   getSessionChatMessages(sessionId, limit = 50) {
     return this.db.query(`SELECT * FROM chat_messages
          WHERE session_id = ?
@@ -5008,6 +5056,39 @@ class MemDatabase {
          ORDER BY created_at_epoch DESC, id DESC
          LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
   }
+  vecChatInsert(chatMessageId, embedding) {
+    if (!this.vecAvailable)
+      return;
+    this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
+  }
+  searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
+    if (!this.vecAvailable)
+      return [];
+    const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
+    const visibilityClause = userId ? " AND c.user_id = ?" : "";
+    const transcriptPreference = `
+      AND (
+        c.source_kind = 'transcript'
+        OR NOT EXISTS (
+          SELECT 1 FROM chat_messages t2
+          WHERE t2.session_id = c.session_id
+            AND t2.source_kind = 'transcript'
+        )
+      )`;
+    if (projectId !== null) {
+      return this.db.query(`SELECT v.chat_message_id, v.distance
+           FROM vec_chat_messages v
+           JOIN chat_messages c ON c.id = v.chat_message_id
+           WHERE v.embedding MATCH ?
+             AND k = ?
+             AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
+    }
+    return this.db.query(`SELECT v.chat_message_id, v.distance
+         FROM vec_chat_messages v
+         JOIN chat_messages c ON c.id = v.chat_message_id
+         WHERE v.embedding MATCH ?
+           AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
+  }
   getTranscriptChatMessage(sessionId, transcriptIndex) {
     return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
   }
@@ -5661,6 +5742,9 @@ function formatInspectHints(context, visibleObservationIds = []) {
   if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentToolEvents?.length ?? 0) > 0 || (context.recentChatMessages?.length ?? 0) > 0) {
     hints.push("activity_feed");
   }
+  if ((context.recentPrompts?.length ?? 0) > 0 || (context.recentChatMessages?.length ?? 0) > 0 || context.observations.length > 0) {
+    hints.push("search_recall");
+  }
   if (context.observations.length > 0) {
     hints.push("memory_console");
   }
@@ -5671,10 +5755,12 @@ function formatInspectHints(context, visibleObservationIds = []) {
   if ((context.recentChatMessages?.length ?? 0) > 0) {
     hints.push("recent_chat");
   }
+  if (hasHookOnlyRecentChat(context)) {
+    hints.push("refresh_chat_recall");
+  }
   if (continuityState !== "fresh") {
     hints.push("recent_chat");
     hints.push("recent_handoffs");
-    hints.push("refresh_chat_recall");
   }
   const unique = Array.from(new Set(hints)).slice(0, 4);
   if (unique.length === 0)
@@ -6144,6 +6230,10 @@ function hasFreshContinuitySignal(context) {
 function getStartupContinuityState(context) {
   return classifyContinuityState(context.recentPrompts?.length ?? 0, context.recentToolEvents?.length ?? 0, context.recentHandoffs?.length ?? 0, context.recentChatMessages?.length ?? 0, context.recentSessions ?? [], context.recentOutcomes?.length ?? 0);
 }
+function hasHookOnlyRecentChat(context) {
+  const recentChat = context.recentChatMessages ?? [];
+  return recentChat.length > 0 && !recentChat.some((message) => message.source_kind === "transcript");
+}
 function observationAgeDays3(obs) {
   const createdAt = new Date(obs.created_at).getTime();
   if (!Number.isFinite(createdAt))

package/dist/hooks/stop.js

@@ -993,6 +993,17 @@ var MIGRATIONS = [
         ON chat_messages(session_id, transcript_index)
         WHERE transcript_index IS NOT NULL;
     `
+  },
+  {
+    version: 18,
+    description: "Add sqlite-vec semantic search for chat recall",
+    condition: (db) => isVecExtensionLoaded(db),
+    sql: `
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+        chat_message_id INTEGER PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -1066,6 +1077,9 @@ function inferLegacySchemaVersion(db) {
   if (columnExists(db, "chat_messages", "source_kind") && columnExists(db, "chat_messages", "transcript_index")) {
     version = Math.max(version, 17);
   }
+  if (tableExists(db, "vec_chat_messages")) {
+    version = Math.max(version, 18);
+  }
   return version;
 }
 function runMigrations(db) {
@@ -1182,6 +1196,20 @@ function ensureChatMessageColumns(db) {
     db.exec("PRAGMA user_version = 17");
   }
 }
+function ensureChatVectorTable(db) {
+  if (!isVecExtensionLoaded(db))
+    return;
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_chat_messages USING vec0(
+      chat_message_id INTEGER PRIMARY KEY,
+      embedding FLOAT[384]
+    );
+  `);
+  const current = getSchemaVersion(db);
+  if (current < 18) {
+    db.exec("PRAGMA user_version = 18");
+  }
+}
 function ensureSyncOutboxSupportsChatMessages(db) {
   if (syncOutboxSupportsChatMessages(db)) {
     const current = getSchemaVersion(db);
@@ -1315,6 +1343,7 @@ class MemDatabase {
     ensureObservationTypes(this.db);
     ensureSessionSummaryColumns(this.db);
     ensureChatMessageColumns(this.db);
+    ensureChatVectorTable(this.db);
     ensureSyncOutboxSupportsChatMessages(this.db);
   }
   loadVecExtension() {
@@ -1681,6 +1710,14 @@ class MemDatabase {
   getChatMessageByRemoteSourceId(remoteSourceId) {
     return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
   }
+  getChatMessagesByIds(ids) {
+    if (ids.length === 0)
+      return [];
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = this.db.query(`SELECT * FROM chat_messages WHERE id IN (${placeholders})`).all(...ids);
+    const order = new Map(ids.map((id, index) => [id, index]));
+    return rows.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
+  }
   getSessionChatMessages(sessionId, limit = 50) {
     return this.db.query(`SELECT * FROM chat_messages
          WHERE session_id = ?
@@ -1757,6 +1794,39 @@ class MemDatabase {
          ORDER BY created_at_epoch DESC, id DESC
          LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
   }
+  vecChatInsert(chatMessageId, embedding) {
+    if (!this.vecAvailable)
+      return;
+    this.db.query("INSERT OR REPLACE INTO vec_chat_messages (chat_message_id, embedding) VALUES (?, ?)").run(chatMessageId, new Uint8Array(embedding.buffer));
+  }
+  searchChatVec(queryEmbedding, projectId, limit = 20, userId) {
+    if (!this.vecAvailable)
+      return [];
+    const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
+    const visibilityClause = userId ? " AND c.user_id = ?" : "";
+    const transcriptPreference = `
+      AND (
+        c.source_kind = 'transcript'
+        OR NOT EXISTS (
+          SELECT 1 FROM chat_messages t2
+          WHERE t2.session_id = c.session_id
+            AND t2.source_kind = 'transcript'
+        )
+      )`;
+    if (projectId !== null) {
+      return this.db.query(`SELECT v.chat_message_id, v.distance
+           FROM vec_chat_messages v
+           JOIN chat_messages c ON c.id = v.chat_message_id
+           WHERE v.embedding MATCH ?
+             AND k = ?
+             AND c.project_id = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, projectId, ...userId ? [userId] : []);
+    }
+    return this.db.query(`SELECT v.chat_message_id, v.distance
+         FROM vec_chat_messages v
+         JOIN chat_messages c ON c.id = v.chat_message_id
+         WHERE v.embedding MATCH ?
+           AND k = ?` + visibilityClause + transcriptPreference).all(embeddingBlob, limit, ...userId ? [userId] : []);
+  }
   getTranscriptChatMessage(sessionId, transcriptIndex) {
     return this.db.query("SELECT * FROM chat_messages WHERE session_id = ? AND transcript_index = ?").get(sessionId, transcriptIndex) ?? null;
   }
@@ -2719,6 +2789,9 @@ function composeEmbeddingText(obs) {
 
 `);
 }
+function composeChatEmbeddingText(text) {
+  return text.replace(/\s+/g, " ").trim().slice(0, 2000);
+}
 async function getPipeline() {
   if (_pipeline)
     return _pipeline;
@@ -3009,7 +3082,7 @@ function buildBeacon(db, config, sessionId, metrics) {
     sentinel_used: valueSignals.security_findings_count > 0,
     risk_score: riskScore,
     stacks_detected: stacks,
-    client_version: "0.4.26",
+    client_version: "0.4.28",
     context_observations_injected: metrics?.contextObsInjected ?? 0,
     context_total_available: metrics?.contextTotalAvailable ?? 0,
     recall_attempts: metrics?.recallAttempts ?? 0,
@@ -3910,7 +3983,7 @@ function readTranscript(sessionId, cwd, transcriptPath) {
   }
   return messages;
 }
-function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
+async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
   const messages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
     ...message,
     text: message.text.trim()
@@ -3940,6 +4013,12 @@ function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
       transcript_index: transcriptIndex
     });
     db.addToOutbox("chat_message", row.id);
+    if (db.vecAvailable) {
+      const embedding = await embedText(composeChatEmbeddingText(message.text));
+      if (embedding) {
+        db.vecChatInsert(row.id, embedding);
+      }
+    }
     imported++;
   }
   return { imported, total: messages.length };
@@ -4040,6 +4119,8 @@ function getSessionStory(db, input) {
     summary,
     prompts,
     chat_messages: chatMessages,
+    chat_source_summary: summarizeChatSources(chatMessages),
+    chat_coverage_state: chatMessages.some((message) => message.source_kind === "transcript") ? "transcript-backed" : chatMessages.length > 0 ? "hook-only" : "none",
     tool_events: toolEvents,
     observations,
     handoffs,
@@ -4137,6 +4218,12 @@ function collectProvenanceSummary(observations) {
   }
   return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => b.count - a.count || a.tool.localeCompare(b.tool)).slice(0, 6);
 }
+function summarizeChatSources(messages) {
+  return messages.reduce((summary, message) => {
+    summary[message.source_kind] += 1;
+    return summary;
+  }, { transcript: 0, hook: 0 });
+}
 
 // src/tools/handoffs.ts
 async function upsertRollingHandoff(db, config, input) {
@@ -4477,7 +4564,7 @@ async function main() {
   try {
     if (event.session_id) {
       db.completeSession(event.session_id);
-      syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path);
+      await syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path);
       if (event.last_assistant_message) {
         try {
           const detected = detectProject(event.cwd);
@@ -4500,6 +4587,12 @@ async function main() {
               source_kind: "hook"
             });
             db.addToOutbox("chat_message", chatMessage.id);
+            if (db.vecAvailable) {
+              const chatEmbedding = await embedText(composeChatEmbeddingText(event.last_assistant_message));
+              if (chatEmbedding) {
+                db.vecChatInsert(chatMessage.id, chatEmbedding);
+              }
+            }
           }
           createAssistantCheckpoint(db, event.session_id, event.cwd, event.last_assistant_message);
         } catch {}