engrm 0.4.11 → 0.4.13

package/README.md

@@ -215,6 +215,8 @@ The MCP server exposes tools that supported agents can call directly:
 | `memory_console` | Show a high-signal local memory console for the current project |
 | `project_memory_index` | Show typed local memory by project, including hot files and recent sessions |
 | `workspace_memory_index` | Show cross-project local memory coverage across the whole workspace |
+| `tool_memory_index` | Show which source tools and plugins are creating durable memory |
+| `session_tool_memory` | Show which tools in one session produced reusable memory and which produced none |
 | `recent_requests` | Inspect captured raw user prompt chronology |
 | `recent_tools` | Inspect captured raw tool chronology |
 | `recent_sessions` | List recent local sessions to inspect further |
@@ -222,6 +224,32 @@ The MCP server exposes tools that supported agents can call directly:
 | `plugin_catalog` | Inspect Engrm plugin manifests for memory-aware integrations |
 | `save_plugin_memory` | Save reduced plugin output with stable Engrm provenance |
 | `capture_git_diff` | Reduce a git diff into a durable memory object and save it |
+| `capture_git_worktree` | Read the current git worktree diff and save reduced memory directly |
+| `capture_repo_scan` | Run a lightweight repo scan and save reduced findings as memory |
+| `capture_openclaw_content` | Save OpenClaw content, research, and follow-up work as plugin memory |
+
+### Thin Tools, Thick Memory
+
+Engrm now has a real thin-tool layer, not just a plugin spec.
+
+Current first-party thin tools:
+
+- `capture_git_worktree`
+  - reads the current repo diff directly
+  - reduces it through `engrm.git-diff`
+- `capture_repo_scan`
+  - runs a lightweight repo scan
+  - reduces it through `engrm.repo-scan`
+- `capture_openclaw_content`
+  - captures posted/researched/outcome-style OpenClaw work
+  - reduces it through `engrm.openclaw-content`
+
+These tools are intentionally small:
+
+- tiny input surface
+- local-first execution
+- reduced durable memory output
+- visible in Engrm's local inspection tools so we can judge tool value honestly
 
 ### Local Memory Inspection
 
@@ -236,8 +264,10 @@ Recommended flow:
 3. activity_feed
 4. recent_sessions
 5. session_story
-6. project_memory_index
-7. workspace_memory_index
+6. tool_memory_index
+7. session_tool_memory
+8. project_memory_index
+9. workspace_memory_index
 ```
 
 What each tool is good for:
@@ -247,9 +277,30 @@ What each tool is good for:
 - `activity_feed` shows the merged chronology across prompts, tools, observations, and summaries
 - `recent_sessions` helps you pick a session worth opening
 - `session_story` reconstructs one session in detail
+- `tool_memory_index` shows which tools and plugins are actually producing durable memory
+- `session_tool_memory` shows which tool calls in one session turned into reusable memory and which did not
 - `project_memory_index` shows typed memory by repo
 - `workspace_memory_index` shows coverage across all repos on the machine
 
+### Thin Tool Workflow
+
+The current practical flow for thin tools is:
+
+```text
+1. memory_console / project_memory_index
+2. tool_memory_index
+3. capture_git_worktree or capture_repo_scan
+4. session_tool_memory
+5. session_story
+```
+
+That lets you:
+
+- see what Engrm already knows
+- see which tools/plugins are producing value
+- capture the current repo state with a thin tool
+- verify whether that tool produced reusable memory
+
 ### Observation Types
 
 | Type | What it captures |

package/dist/cli.js

@@ -597,6 +597,18 @@ var MIGRATIONS = [
       CREATE INDEX IF NOT EXISTS idx_tool_events_created
         ON tool_events(created_at_epoch DESC, id DESC);
     `
+  },
+  {
+    version: 11,
+    description: "Add observation provenance from tool and prompt chronology",
+    sql: `
+      ALTER TABLE observations ADD COLUMN source_tool TEXT;
+      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
+      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
+        ON observations(source_tool, created_at_epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
+        ON observations(session_id, source_prompt_number DESC);
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -650,6 +662,8 @@ function inferLegacySchemaVersion(db) {
     version = Math.max(version, 9);
   if (tableExists(db, "tool_events"))
     version = Math.max(version, 10);
+  if (columnExists(db, "observations", "source_tool"))
+    version = Math.max(version, 11);
   return version;
 }
 function runMigrations(db) {
@@ -736,6 +750,86 @@ var LATEST_SCHEMA_VERSION = MIGRATIONS.filter((m) => !m.condition).reduce((max,
 
 // src/storage/sqlite.ts
 import { createHash as createHash2 } from "node:crypto";
+
+// src/intelligence/summary-sections.ts
+function extractSummaryItems(section, limit) {
+  if (!section || !section.trim())
+    return [];
+  const rawLines = section.split(`
+`).map((line) => line.replace(/\s+/g, " ").trim()).filter(Boolean);
+  const items = [];
+  const seen = new Set;
+  let heading = null;
+  for (const rawLine of rawLines) {
+    const line = stripSectionPrefix(rawLine);
+    if (!line)
+      continue;
+    const headingOnly = parseHeading(line);
+    if (headingOnly) {
+      heading = headingOnly;
+      continue;
+    }
+    const isBullet = /^[-*•]\s+/.test(line);
+    const stripped = line.replace(/^[-*•]\s+/, "").trim();
+    if (!stripped)
+      continue;
+    const item = heading && isBullet ? `${heading}: ${stripped}` : stripped;
+    const normalized = normalizeItem(item);
+    if (!normalized || seen.has(normalized))
+      continue;
+    seen.add(normalized);
+    items.push(item);
+    if (limit && items.length >= limit)
+      break;
+  }
+  return items;
+}
+function formatSummaryItems(section, maxLen) {
+  const items = extractSummaryItems(section);
+  if (items.length === 0)
+    return null;
+  const cleaned = items.map((item) => `- ${item}`).join(`
+`);
+  if (cleaned.length <= maxLen)
+    return cleaned;
+  const truncated = cleaned.slice(0, maxLen).trimEnd();
+  const lastBreak = Math.max(truncated.lastIndexOf(`
+`), truncated.lastIndexOf(" "));
+  const safe = lastBreak > maxLen * 0.5 ? truncated.slice(0, lastBreak) : truncated;
+  return `${safe.trimEnd()}…`;
+}
+function normalizeSummarySection(section) {
+  const items = extractSummaryItems(section);
+  if (items.length === 0) {
+    const cleaned = section?.replace(/\s+/g, " ").trim() ?? "";
+    return cleaned || null;
+  }
+  return items.map((item) => `- ${item}`).join(`
+`);
+}
+function normalizeSummaryRequest(value) {
+  const cleaned = value?.replace(/\s+/g, " ").trim() ?? "";
+  return cleaned || null;
+}
+function stripSectionPrefix(value) {
+  return value.replace(/^(request|investigated|learned|completed|next steps|summary):\s*/i, "").trim();
+}
+function parseHeading(value) {
+  const boldMatch = value.match(/^\*{1,2}\s*(.+?)\s*:\*{1,2}$/);
+  if (boldMatch?.[1]) {
+    return boldMatch[1].trim().replace(/\s+/g, " ");
+  }
+  const plainMatch = value.match(/^(.+?):$/);
+  if (plainMatch?.[1]) {
+    return plainMatch[1].trim().replace(/\s+/g, " ");
+  }
+  return null;
+}
+function normalizeItem(value) {
+  return value.toLowerCase().replace(/\*+/g, "").replace(/\s+/g, " ").trim();
+}
+
+// src/storage/sqlite.ts
 var IS_BUN = typeof globalThis.Bun !== "undefined";
 function openDatabase(dbPath) {
   if (IS_BUN) {
@@ -772,6 +866,7 @@ function openNodeDatabase(dbPath) {
   const BetterSqlite3 = __require("better-sqlite3");
   const raw = new BetterSqlite3(dbPath);
   return {
+    __raw: raw,
     query(sql) {
       const stmt = raw.prepare(sql);
       return {
@@ -809,7 +904,7 @@ class MemDatabase {
   loadVecExtension() {
     try {
       const sqliteVec = __require("sqlite-vec");
-      sqliteVec.load(this.db);
+      sqliteVec.load(this.db.__raw ?? this.db);
       return true;
     } catch {
       return false;
@@ -850,8 +945,9 @@ class MemDatabase {
     const result = this.db.query(`INSERT INTO observations (
           session_id, project_id, type, title, narrative, facts, concepts,
           files_read, files_modified, quality, lifecycle, sensitivity,
-          user_id, device_id, agent, created_at, created_at_epoch
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", createdAt, now);
+          user_id, device_id, agent, source_tool, source_prompt_number,
+          created_at, created_at_epoch
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", obs.source_tool ?? null, obs.source_prompt_number ?? null, createdAt, now);
     const id = Number(result.lastInsertRowid);
     const row = this.getObservationById(id);
     this.ftsInsert(row);
@@ -1092,6 +1188,13 @@ class MemDatabase {
          ORDER BY prompt_number ASC
          LIMIT ?`).all(sessionId, limit);
   }
+  getLatestSessionPromptNumber(sessionId) {
+    const row = this.db.query(`SELECT prompt_number FROM user_prompts
+         WHERE session_id = ?
+         ORDER BY prompt_number DESC
+         LIMIT 1`).get(sessionId);
+    return row?.prompt_number ?? null;
+  }
   insertToolEvent(input) {
     const createdAt = input.created_at_epoch ?? Math.floor(Date.now() / 1000);
     const result = this.db.query(`INSERT INTO tool_events (
@@ -1201,8 +1304,15 @@ class MemDatabase {
   }
   insertSessionSummary(summary) {
     const now = Math.floor(Date.now() / 1000);
+    const normalized = {
+      request: normalizeSummaryRequest(summary.request),
+      investigated: normalizeSummarySection(summary.investigated),
+      learned: normalizeSummarySection(summary.learned),
+      completed: normalizeSummarySection(summary.completed),
+      next_steps: normalizeSummarySection(summary.next_steps)
+    };
     const result = this.db.query(`INSERT INTO session_summaries (session_id, project_id, user_id, request, investigated, learned, completed, next_steps, created_at_epoch)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, summary.request, summary.investigated, summary.learned, summary.completed, summary.next_steps, now);
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, now);
     const id = Number(result.lastInsertRowid);
     return this.db.query("SELECT * FROM session_summaries WHERE id = ?").get(id);
   }
@@ -1707,6 +1817,18 @@ var MIGRATIONS2 = [
       CREATE INDEX IF NOT EXISTS idx_tool_events_created
         ON tool_events(created_at_epoch DESC, id DESC);
     `
+  },
+  {
+    version: 11,
+    description: "Add observation provenance from tool and prompt chronology",
+    sql: `
+      ALTER TABLE observations ADD COLUMN source_tool TEXT;
+      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
+      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
+        ON observations(source_tool, created_at_epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
+        ON observations(session_id, source_prompt_number DESC);
+    `
   }
 ];
 function isVecExtensionLoaded2(db) {
@@ -2986,6 +3108,7 @@ async function saveObservation(db, config, input) {
       reason: `Merged into existing observation #${duplicate.id}`
     };
   }
+  const sourcePromptNumber = input.source_prompt_number ?? (input.session_id ? db.getLatestSessionPromptNumber(input.session_id) : null);
   const obs = db.insertObservation({
     session_id: input.session_id ?? null,
     project_id: project.id,
@@ -3001,7 +3124,9 @@ async function saveObservation(db, config, input) {
     sensitivity,
     user_id: config.user_id,
     device_id: config.device_id,
-    agent: input.agent ?? "claude-code"
+    agent: input.agent ?? "claude-code",
+    source_tool: input.source_tool ?? null,
+    source_prompt_number: sourcePromptNumber
   });
   db.addToOutbox("observation", obs.id);
   if (db.vecAvailable) {
@@ -4000,7 +4125,12 @@ function handleUpdate() {
   console.log(`Updating Engrm to latest version...
 `);
   try {
-    execSync2("npm install -g engrm@latest", { stdio: "inherit" });
+    const latest = execSync2("npm view engrm version", { encoding: "utf-8" }).trim();
+    if (!latest)
+      throw new Error("Could not resolve latest engrm version from npm");
+    console.log(`Installing engrm@${latest}...
+`);
+    execSync2(`npm install -g engrm@${latest}`, { stdio: "inherit" });
     console.log(`
 Update complete. Re-registering integrations...`);
     const result = registerAll();
@@ -4011,7 +4141,8 @@ Update complete. Re-registering integrations...`);
     console.log(`
 Restart Claude Code or Codex to use the new version.`);
   } catch (error) {
-    console.error("Update failed. Try manually: npm install -g engrm@latest");
+    console.error(`Update failed: ${error instanceof Error ? error.message : String(error)}`);
+    console.error("Try manually: npm install -g engrm@<version>");
   }
 }
 async function handleDoctor() {

package/dist/hooks/elicitation-result.js

@@ -797,6 +797,7 @@ async function saveObservation(db, config, input) {
       reason: `Merged into existing observation #${duplicate.id}`
     };
   }
+  const sourcePromptNumber = input.source_prompt_number ?? (input.session_id ? db.getLatestSessionPromptNumber(input.session_id) : null);
   const obs = db.insertObservation({
     session_id: input.session_id ?? null,
     project_id: project.id,
@@ -812,7 +813,9 @@ async function saveObservation(db, config, input) {
     sensitivity,
     user_id: config.user_id,
     device_id: config.device_id,
-    agent: input.agent ?? "claude-code"
+    agent: input.agent ?? "claude-code",
+    source_tool: input.source_tool ?? null,
+    source_prompt_number: sourcePromptNumber
   });
   db.addToOutbox("observation", obs.id);
   if (db.vecAvailable) {
@@ -1427,6 +1430,18 @@ var MIGRATIONS = [
       CREATE INDEX IF NOT EXISTS idx_tool_events_created
         ON tool_events(created_at_epoch DESC, id DESC);
     `
+  },
+  {
+    version: 11,
+    description: "Add observation provenance from tool and prompt chronology",
+    sql: `
+      ALTER TABLE observations ADD COLUMN source_tool TEXT;
+      ALTER TABLE observations ADD COLUMN source_prompt_number INTEGER;
+      CREATE INDEX IF NOT EXISTS idx_observations_source_tool
+        ON observations(source_tool, created_at_epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_observations_source_prompt
+        ON observations(session_id, source_prompt_number DESC);
+    `
   }
 ];
 function isVecExtensionLoaded(db) {
@@ -1480,6 +1495,8 @@ function inferLegacySchemaVersion(db) {
     version = Math.max(version, 9);
   if (tableExists(db, "tool_events"))
     version = Math.max(version, 10);
+  if (columnExists(db, "observations", "source_tool"))
+    version = Math.max(version, 11);
   return version;
 }
 function runMigrations(db) {
@@ -1562,6 +1579,86 @@ var LATEST_SCHEMA_VERSION = MIGRATIONS.filter((m) => !m.condition).reduce((max,
 
 // src/storage/sqlite.ts
 import { createHash as createHash2 } from "node:crypto";
+
+// src/intelligence/summary-sections.ts
+function extractSummaryItems(section, limit) {
+  if (!section || !section.trim())
+    return [];
+  const rawLines = section.split(`
+`).map((line) => line.replace(/\s+/g, " ").trim()).filter(Boolean);
+  const items = [];
+  const seen = new Set;
+  let heading = null;
+  for (const rawLine of rawLines) {
+    const line = stripSectionPrefix(rawLine);
+    if (!line)
+      continue;
+    const headingOnly = parseHeading(line);
+    if (headingOnly) {
+      heading = headingOnly;
+      continue;
+    }
+    const isBullet = /^[-*•]\s+/.test(line);
+    const stripped = line.replace(/^[-*•]\s+/, "").trim();
+    if (!stripped)
+      continue;
+    const item = heading && isBullet ? `${heading}: ${stripped}` : stripped;
+    const normalized = normalizeItem(item);
+    if (!normalized || seen.has(normalized))
+      continue;
+    seen.add(normalized);
+    items.push(item);
+    if (limit && items.length >= limit)
+      break;
+  }
+  return items;
+}
+function formatSummaryItems(section, maxLen) {
+  const items = extractSummaryItems(section);
+  if (items.length === 0)
+    return null;
+  const cleaned = items.map((item) => `- ${item}`).join(`
+`);
+  if (cleaned.length <= maxLen)
+    return cleaned;
+  const truncated = cleaned.slice(0, maxLen).trimEnd();
+  const lastBreak = Math.max(truncated.lastIndexOf(`
+`), truncated.lastIndexOf(" "));
+  const safe = lastBreak > maxLen * 0.5 ? truncated.slice(0, lastBreak) : truncated;
+  return `${safe.trimEnd()}…`;
+}
+function normalizeSummarySection(section) {
+  const items = extractSummaryItems(section);
+  if (items.length === 0) {
+    const cleaned = section?.replace(/\s+/g, " ").trim() ?? "";
+    return cleaned || null;
+  }
+  return items.map((item) => `- ${item}`).join(`
+`);
+}
+function normalizeSummaryRequest(value) {
+  const cleaned = value?.replace(/\s+/g, " ").trim() ?? "";
+  return cleaned || null;
+}
+function stripSectionPrefix(value) {
+  return value.replace(/^(request|investigated|learned|completed|next steps|summary):\s*/i, "").trim();
+}
+function parseHeading(value) {
+  const boldMatch = value.match(/^\*{1,2}\s*(.+?)\s*:\*{1,2}$/);
+  if (boldMatch?.[1]) {
+    return boldMatch[1].trim().replace(/\s+/g, " ");
+  }
+  const plainMatch = value.match(/^(.+?):$/);
+  if (plainMatch?.[1]) {
+    return plainMatch[1].trim().replace(/\s+/g, " ");
+  }
+  return null;
+}
+function normalizeItem(value) {
+  return value.toLowerCase().replace(/\*+/g, "").replace(/\s+/g, " ").trim();
+}
+
+// src/storage/sqlite.ts
 var IS_BUN = typeof globalThis.Bun !== "undefined";
 function openDatabase(dbPath) {
   if (IS_BUN) {
@@ -1598,6 +1695,7 @@ function openNodeDatabase(dbPath) {
   const BetterSqlite3 = __require("better-sqlite3");
   const raw = new BetterSqlite3(dbPath);
   return {
+    __raw: raw,
     query(sql) {
       const stmt = raw.prepare(sql);
       return {
@@ -1635,7 +1733,7 @@ class MemDatabase {
   loadVecExtension() {
     try {
       const sqliteVec = __require("sqlite-vec");
-      sqliteVec.load(this.db);
+      sqliteVec.load(this.db.__raw ?? this.db);
       return true;
     } catch {
       return false;
@@ -1676,8 +1774,9 @@ class MemDatabase {
     const result = this.db.query(`INSERT INTO observations (
           session_id, project_id, type, title, narrative, facts, concepts,
           files_read, files_modified, quality, lifecycle, sensitivity,
-          user_id, device_id, agent, created_at, created_at_epoch
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", createdAt, now);
+          user_id, device_id, agent, source_tool, source_prompt_number,
+          created_at, created_at_epoch
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", obs.source_tool ?? null, obs.source_prompt_number ?? null, createdAt, now);
     const id = Number(result.lastInsertRowid);
     const row = this.getObservationById(id);
     this.ftsInsert(row);
@@ -1918,6 +2017,13 @@ class MemDatabase {
          ORDER BY prompt_number ASC
          LIMIT ?`).all(sessionId, limit);
   }
+  getLatestSessionPromptNumber(sessionId) {
+    const row = this.db.query(`SELECT prompt_number FROM user_prompts
+         WHERE session_id = ?
+         ORDER BY prompt_number DESC
+         LIMIT 1`).get(sessionId);
+    return row?.prompt_number ?? null;
+  }
   insertToolEvent(input) {
     const createdAt = input.created_at_epoch ?? Math.floor(Date.now() / 1000);
     const result = this.db.query(`INSERT INTO tool_events (
@@ -2027,8 +2133,15 @@ class MemDatabase {
   }
   insertSessionSummary(summary) {
     const now = Math.floor(Date.now() / 1000);
+    const normalized = {
+      request: normalizeSummaryRequest(summary.request),
+      investigated: normalizeSummarySection(summary.investigated),
+      learned: normalizeSummarySection(summary.learned),
+      completed: normalizeSummarySection(summary.completed),
+      next_steps: normalizeSummarySection(summary.next_steps)
+    };
     const result = this.db.query(`INSERT INTO session_summaries (session_id, project_id, user_id, request, investigated, learned, completed, next_steps, created_at_epoch)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, summary.request, summary.investigated, summary.learned, summary.completed, summary.next_steps, now);
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, normalized.request, normalized.investigated, normalized.learned, normalized.completed, normalized.next_steps, now);
     const id = Number(result.lastInsertRowid);
     return this.db.query("SELECT * FROM session_summaries WHERE id = ?").get(id);
   }