engrm 0.4.21 → 0.4.23

package/README.md

@@ -221,6 +221,9 @@ The MCP server exposes tools that supported agents can call directly:
 | `recent_tools` | Inspect captured raw tool chronology |
 | `recent_sessions` | List recent local sessions to inspect further |
 | `session_story` | Show prompts, tools, observations, and summary for one session |
+| `create_handoff` | Save an explicit syncable handoff so you can resume work on another device |
+| `recent_handoffs` | List recent saved handoffs for the current project or workspace |
+| `load_handoff` | Open a saved handoff as a resume point for a new session |
 | `plugin_catalog` | Inspect Engrm plugin manifests for memory-aware integrations |
 | `save_plugin_memory` | Save reduced plugin output with stable Engrm provenance |
 | `capture_git_diff` | Reduce a git diff into a durable memory object and save it |
@@ -273,6 +276,21 @@ These tools are intentionally small:
 - reduced durable memory output
 - visible in Engrm's local inspection tools so we can judge tool value honestly
 
+### Explicit Handoffs
+
+For long-running work across devices, Engrm now has an explicit handoff flow:
+
+- `create_handoff`
+  - snapshot the active thread into a syncable handoff message
+- `recent_handoffs`
+  - list the latest saved handoffs
+- `load_handoff`
+  - reopen a saved handoff as a clear resume point in a new session
+
+This is the deliberate version of multi-device continuity: useful when you want to move from laptop to home machine without waiting for an end-of-session summary.
+
+The separate chat lane is still kept distinct from durable observations, but it can now sync too, so recent user/assistant conversation is recoverable on another machine without polluting the main memory feed.
+
 ### Local Memory Inspection
 
 For local testing, Engrm now exposes a small inspection set that lets you see

package/dist/cli.js

@@ -305,7 +305,7 @@ var MIGRATIONS = [
       -- Sync outbox (offline-first queue)
       CREATE TABLE sync_outbox (
           id              INTEGER PRIMARY KEY AUTOINCREMENT,
-          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary')),
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary', 'chat_message')),
           record_id       INTEGER NOT NULL,
           status          TEXT DEFAULT 'pending' CHECK (status IN (
               'pending', 'syncing', 'synced', 'failed'
@@ -600,6 +600,18 @@ var MIGRATIONS = [
   },
   {
     version: 11,
+    description: "Add observation provenance from tool and prompt chronology",
+    sql: `
+      ALTER TABLE observations ADD COLUMN source_tool TEXT;
+      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
+      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
+        ON observations(source_tool, created_at_epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
+        ON observations(session_id, source_prompt_number DESC);
+    `
+  },
+  {
+    version: 12,
     description: "Add synced handoff metadata to session summaries",
     sql: `
       ALTER TABLE session_summaries ADD COLUMN capture_state TEXT;
@@ -609,15 +621,79 @@ var MIGRATIONS = [
     `
   },
   {
-    version: 11,
-    description: "Add observation provenance from tool and prompt chronology",
+    version: 13,
+    description: "Add current_thread to session summaries",
     sql: `
-      ALTER TABLE observations ADD COLUMN source_tool TEXT;
-      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
-      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
-        ON observations(source_tool, created_at_epoch DESC);
-      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
-        ON observations(session_id, source_prompt_number DESC);
+      ALTER TABLE session_summaries ADD COLUMN current_thread TEXT;
+    `
+  },
+  {
+    version: 14,
+    description: "Add chat_messages lane for raw conversation recall",
+    sql: `
+      CREATE TABLE IF NOT EXISTS chat_messages (
+          id                INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id        TEXT NOT NULL,
+          project_id        INTEGER REFERENCES projects(id),
+          role              TEXT NOT NULL CHECK (role IN ('user', 'assistant')),
+          content           TEXT NOT NULL,
+          user_id           TEXT NOT NULL,
+          device_id         TEXT NOT NULL,
+          agent             TEXT DEFAULT 'claude-code',
+          created_at_epoch  INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_session
+        ON chat_messages(session_id, created_at_epoch DESC, id DESC);
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_project
+        ON chat_messages(project_id, created_at_epoch DESC, id DESC);
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_created
+        ON chat_messages(created_at_epoch DESC, id DESC);
+    `
+  },
+  {
+    version: 15,
+    description: "Add remote_source_id for chat message sync deduplication",
+    sql: `
+      ALTER TABLE chat_messages ADD COLUMN remote_source_id TEXT;
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_chat_messages_remote_source
+        ON chat_messages(remote_source_id)
+        WHERE remote_source_id IS NOT NULL;
+    `
+  },
+  {
+    version: 16,
+    description: "Allow chat_message records in sync_outbox",
+    sql: `
+      CREATE TABLE sync_outbox_new (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary', 'chat_message')),
+          record_id       INTEGER NOT NULL,
+          status          TEXT DEFAULT 'pending' CHECK (status IN (
+              'pending', 'syncing', 'synced', 'failed'
+          )),
+          retry_count     INTEGER DEFAULT 0,
+          max_retries     INTEGER DEFAULT 10,
+          last_error      TEXT,
+          created_at_epoch INTEGER NOT NULL,
+          synced_at_epoch  INTEGER,
+          next_retry_epoch INTEGER
+      );
+
+      INSERT INTO sync_outbox_new (
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      )
+      SELECT
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      FROM sync_outbox;
+
+      DROP TABLE sync_outbox;
+      ALTER TABLE sync_outbox_new RENAME TO sync_outbox;
+
+      CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
+      CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
     `
   }
 ];
@@ -674,6 +750,21 @@ function inferLegacySchemaVersion(db) {
     version = Math.max(version, 10);
   if (columnExists(db, "observations", "source_tool"))
     version = Math.max(version, 11);
+  if (columnExists(db, "session_summaries", "capture_state") && columnExists(db, "session_summaries", "recent_tool_names") && columnExists(db, "session_summaries", "hot_files") && columnExists(db, "session_summaries", "recent_outcomes")) {
+    version = Math.max(version, 12);
+  }
+  if (columnExists(db, "session_summaries", "current_thread")) {
+    version = Math.max(version, 13);
+  }
+  if (tableExists(db, "chat_messages")) {
+    version = Math.max(version, 14);
+  }
+  if (columnExists(db, "chat_messages", "remote_source_id")) {
+    version = Math.max(version, 15);
+  }
+  if (syncOutboxSupportsChatMessages(db)) {
+    version = Math.max(version, 16);
+  }
   return version;
 }
 function runMigrations(db) {
@@ -752,6 +843,89 @@ function ensureObservationTypes(db) {
     }
   }
 }
+function ensureSessionSummaryColumns(db) {
+  const required = [
+    "capture_state",
+    "recent_tool_names",
+    "hot_files",
+    "recent_outcomes",
+    "current_thread"
+  ];
+  for (const column of required) {
+    if (columnExists(db, "session_summaries", column))
+      continue;
+    db.exec(`ALTER TABLE session_summaries ADD COLUMN ${column} TEXT`);
+  }
+  const current = getSchemaVersion(db);
+  if (current < 13) {
+    db.exec("PRAGMA user_version = 13");
+  }
+}
+function ensureChatMessageColumns(db) {
+  if (!tableExists(db, "chat_messages"))
+    return;
+  if (!columnExists(db, "chat_messages", "remote_source_id")) {
+    db.exec("ALTER TABLE chat_messages ADD COLUMN remote_source_id TEXT");
+  }
+  db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_chat_messages_remote_source ON chat_messages(remote_source_id) WHERE remote_source_id IS NOT NULL");
+  const current = getSchemaVersion(db);
+  if (current < 15) {
+    db.exec("PRAGMA user_version = 15");
+  }
+}
+function ensureSyncOutboxSupportsChatMessages(db) {
+  if (syncOutboxSupportsChatMessages(db)) {
+    const current = getSchemaVersion(db);
+    if (current < 16) {
+      db.exec("PRAGMA user_version = 16");
+    }
+    return;
+  }
+  db.exec("BEGIN TRANSACTION");
+  try {
+    db.exec(`
+      CREATE TABLE sync_outbox_new (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary', 'chat_message')),
+          record_id       INTEGER NOT NULL,
+          status          TEXT DEFAULT 'pending' CHECK (status IN (
+              'pending', 'syncing', 'synced', 'failed'
+          )),
+          retry_count     INTEGER DEFAULT 0,
+          max_retries     INTEGER DEFAULT 10,
+          last_error      TEXT,
+          created_at_epoch INTEGER NOT NULL,
+          synced_at_epoch  INTEGER,
+          next_retry_epoch INTEGER
+      );
+
+      INSERT INTO sync_outbox_new (
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      )
+      SELECT
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      FROM sync_outbox;
+
+      DROP TABLE sync_outbox;
+      ALTER TABLE sync_outbox_new RENAME TO sync_outbox;
+
+      CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
+      CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
+    `);
+    db.exec("PRAGMA user_version = 16");
+    db.exec("COMMIT");
+  } catch (error) {
+    db.exec("ROLLBACK");
+    throw new Error(`sync_outbox repair failed: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+function syncOutboxSupportsChatMessages(db) {
+  const row = db.query("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get("sync_outbox");
+  const sql = row?.sql ?? "";
+  return sql.includes("'chat_message'");
+}
 function getSchemaVersion(db) {
   const result = db.query("PRAGMA user_version").get();
   return result.user_version;
@@ -910,6 +1084,9 @@ class MemDatabase {
     this.vecAvailable = this.loadVecExtension();
     runMigrations(this.db);
     ensureObservationTypes(this.db);
+    ensureSessionSummaryColumns(this.db);
+    ensureChatMessageColumns(this.db);
+    ensureSyncOutboxSupportsChatMessages(this.db);
   }
   loadVecExtension() {
     try {
@@ -1135,6 +1312,7 @@ class MemDatabase {
              p.name AS project_name,
              ss.request AS request,
              ss.completed AS completed,
+             ss.current_thread AS current_thread,
              ss.capture_state AS capture_state,
              ss.recent_tool_names AS recent_tool_names,
              ss.hot_files AS hot_files,
@@ -1153,6 +1331,7 @@ class MemDatabase {
            p.name AS project_name,
            ss.request AS request,
            ss.completed AS completed,
+           ss.current_thread AS current_thread,
            ss.capture_state AS capture_state,
            ss.recent_tool_names AS recent_tool_names,
            ss.hot_files AS hot_files,
@@ -1243,6 +1422,54 @@ class MemDatabase {
          ORDER BY created_at_epoch DESC, id DESC
          LIMIT ?`).all(...userId ? [userId] : [], limit);
   }
+  insertChatMessage(input) {
+    const createdAt = input.created_at_epoch ?? Math.floor(Date.now() / 1000);
+    const content = input.content.trim();
+    const result = this.db.query(`INSERT INTO chat_messages (
+          session_id, project_id, role, content, user_id, device_id, agent, created_at_epoch, remote_source_id
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(input.session_id, input.project_id, input.role, content, input.user_id, input.device_id, input.agent ?? "claude-code", createdAt, input.remote_source_id ?? null);
+    return this.getChatMessageById(Number(result.lastInsertRowid));
+  }
+  getChatMessageById(id) {
+    return this.db.query("SELECT * FROM chat_messages WHERE id = ?").get(id) ?? null;
+  }
+  getChatMessageByRemoteSourceId(remoteSourceId) {
+    return this.db.query("SELECT * FROM chat_messages WHERE remote_source_id = ?").get(remoteSourceId) ?? null;
+  }
+  getSessionChatMessages(sessionId, limit = 50) {
+    return this.db.query(`SELECT * FROM chat_messages
+         WHERE session_id = ?
+         ORDER BY created_at_epoch ASC, id ASC
+         LIMIT ?`).all(sessionId, limit);
+  }
+  getRecentChatMessages(projectId, limit = 20, userId) {
+    const visibilityClause = userId ? " AND user_id = ?" : "";
+    if (projectId !== null) {
+      return this.db.query(`SELECT * FROM chat_messages
+           WHERE project_id = ?${visibilityClause}
+           ORDER BY created_at_epoch DESC, id DESC
+           LIMIT ?`).all(projectId, ...userId ? [userId] : [], limit);
+    }
+    return this.db.query(`SELECT * FROM chat_messages
+         WHERE 1 = 1${visibilityClause}
+         ORDER BY created_at_epoch DESC, id DESC
+         LIMIT ?`).all(...userId ? [userId] : [], limit);
+  }
+  searchChatMessages(query, projectId, limit = 20, userId) {
+    const needle = `%${query.toLowerCase()}%`;
+    const visibilityClause = userId ? " AND user_id = ?" : "";
+    if (projectId !== null) {
+      return this.db.query(`SELECT * FROM chat_messages
+           WHERE project_id = ?
+             AND lower(content) LIKE ?${visibilityClause}
+           ORDER BY created_at_epoch DESC, id DESC
+           LIMIT ?`).all(projectId, needle, ...userId ? [userId] : [], limit);
+    }
+    return this.db.query(`SELECT * FROM chat_messages
+         WHERE lower(content) LIKE ?${visibilityClause}
+         ORDER BY created_at_epoch DESC, id DESC
+         LIMIT ?`).all(needle, ...userId ? [userId] : [], limit);
+  }
   addToOutbox(recordType, recordId) {
     const now = Math.floor(Date.now() / 1000);
     this.db.query(`INSERT INTO sync_outbox (record_type, record_id, created_at_epoch)
@@ -1331,9 +1558,9 @@ class MemDatabase {
     };
     const result = this.db.query(`INSERT INTO session_summaries (
           session_id, project_id, user_id, request, investigated, learned, completed, next_steps,
-          capture_state, recent_tool_names, hot_files, recent_outcomes, created_at_epoch
+          current_thread, capture_state, recent_tool_names, hot_files, recent_outcomes, created_at_epoch
         )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, summary.capture_state ?? null, summary.recent_tool_names ?? null, summary.hot_files ?? null, summary.recent_outcomes ?? null, now);
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, summary.current_thread ?? null, summary.capture_state ?? null, summary.recent_tool_names ?? null, summary.hot_files ?? null, summary.recent_outcomes ?? null, now);
     const id = Number(result.lastInsertRowid);
     return this.db.query("SELECT * FROM session_summaries WHERE id = ?").get(id);
   }
@@ -1349,6 +1576,7 @@ class MemDatabase {
       learned: normalizeSummarySection(summary.learned ?? existing.learned),
       completed: normalizeSummarySection(summary.completed ?? existing.completed),
       next_steps: normalizeSummarySection(summary.next_steps ?? existing.next_steps),
+      current_thread: summary.current_thread ?? existing.current_thread,
       capture_state: summary.capture_state ?? existing.capture_state,
       recent_tool_names: summary.recent_tool_names ?? existing.recent_tool_names,
       hot_files: summary.hot_files ?? existing.hot_files,
@@ -1362,12 +1590,13 @@ class MemDatabase {
              learned = ?,
              completed = ?,
              next_steps = ?,
+             current_thread = ?,
              capture_state = ?,
              recent_tool_names = ?,
              hot_files = ?,
              recent_outcomes = ?,
              created_at_epoch = ?
-         WHERE session_id = ?`).run(summary.project_id ?? existing.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, normalized.capture_state, normalized.recent_tool_names, normalized.hot_files, normalized.recent_outcomes, now, summary.session_id);
+         WHERE session_id = ?`).run(summary.project_id ?? existing.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, normalized.current_thread, normalized.capture_state, normalized.recent_tool_names, normalized.hot_files, normalized.recent_outcomes, now, summary.session_id);
     return this.getSessionSummary(summary.session_id);
   }
   getSessionSummary(sessionId) {
@@ -1579,7 +1808,7 @@ var MIGRATIONS2 = [
       -- Sync outbox (offline-first queue)
       CREATE TABLE sync_outbox (
           id              INTEGER PRIMARY KEY AUTOINCREMENT,
-          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary')),
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary', 'chat_message')),
           record_id       INTEGER NOT NULL,
           status          TEXT DEFAULT 'pending' CHECK (status IN (
               'pending', 'syncing', 'synced', 'failed'
@@ -1874,6 +2103,18 @@ var MIGRATIONS2 = [
   },
   {
     version: 11,
+    description: "Add observation provenance from tool and prompt chronology",
+    sql: `
+      ALTER TABLE observations ADD COLUMN source_tool TEXT;
+      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
+      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
+        ON observations(source_tool, created_at_epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
+        ON observations(session_id, source_prompt_number DESC);
+    `
+  },
+  {
+    version: 12,
     description: "Add synced handoff metadata to session summaries",
     sql: `
       ALTER TABLE session_summaries ADD COLUMN capture_state TEXT;
@@ -1883,15 +2124,79 @@ var MIGRATIONS2 = [
     `
   },
   {
-    version: 11,
-    description: "Add observation provenance from tool and prompt chronology",
+    version: 13,
+    description: "Add current_thread to session summaries",
     sql: `
-      ALTER TABLE observations ADD COLUMN source_tool TEXT;
-      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
-      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
-        ON observations(source_tool, created_at_epoch DESC);
-      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
-        ON observations(session_id, source_prompt_number DESC);
+      ALTER TABLE session_summaries ADD COLUMN current_thread TEXT;
+    `
+  },
+  {
+    version: 14,
+    description: "Add chat_messages lane for raw conversation recall",
+    sql: `
+      CREATE TABLE IF NOT EXISTS chat_messages (
+          id                INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id        TEXT NOT NULL,
+          project_id        INTEGER REFERENCES projects(id),
+          role              TEXT NOT NULL CHECK (role IN ('user', 'assistant')),
+          content           TEXT NOT NULL,
+          user_id           TEXT NOT NULL,
+          device_id         TEXT NOT NULL,
+          agent             TEXT DEFAULT 'claude-code',
+          created_at_epoch  INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_session
+        ON chat_messages(session_id, created_at_epoch DESC, id DESC);
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_project
+        ON chat_messages(project_id, created_at_epoch DESC, id DESC);
+      CREATE INDEX IF NOT EXISTS idx_chat_messages_created
+        ON chat_messages(created_at_epoch DESC, id DESC);
+    `
+  },
+  {
+    version: 15,
+    description: "Add remote_source_id for chat message sync deduplication",
+    sql: `
+      ALTER TABLE chat_messages ADD COLUMN remote_source_id TEXT;
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_chat_messages_remote_source
+        ON chat_messages(remote_source_id)
+        WHERE remote_source_id IS NOT NULL;
+    `
+  },
+  {
+    version: 16,
+    description: "Allow chat_message records in sync_outbox",
+    sql: `
+      CREATE TABLE sync_outbox_new (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary', 'chat_message')),
+          record_id       INTEGER NOT NULL,
+          status          TEXT DEFAULT 'pending' CHECK (status IN (
+              'pending', 'syncing', 'synced', 'failed'
+          )),
+          retry_count     INTEGER DEFAULT 0,
+          max_retries     INTEGER DEFAULT 10,
+          last_error      TEXT,
+          created_at_epoch INTEGER NOT NULL,
+          synced_at_epoch  INTEGER,
+          next_retry_epoch INTEGER
+      );
+
+      INSERT INTO sync_outbox_new (
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      )
+      SELECT
+        id, record_type, record_id, status, retry_count, max_retries,
+        last_error, created_at_epoch, synced_at_epoch, next_retry_epoch
+      FROM sync_outbox;
+
+      DROP TABLE sync_outbox;
+      ALTER TABLE sync_outbox_new RENAME TO sync_outbox;
+
+      CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
+      CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
     `
   }
 ];