dsh-engram 0.7.2 → 0.8.6

package/lib/approvals.js

@@ -0,0 +1,84 @@
+/**
+ * v0.8-D: the approval gate for artifacts that leave the memory store.
+ *
+ * Why this exists: v0.8-A made a procedure promotable by evidence, and v0.8-B
+ * turns promoted procedures into files in `.dsh/skills` that the host
+ * advertises to every agent. Evidence proves a command *worked*; it does not
+ * prove the instruction is *sound* — three successful runs of a harmful recipe
+ * are still three successful runs. So the two surfaces are gated differently on
+ * purpose:
+ *
+ *   - the [ENGRAM] index line is cheap, reversible and private → evidence gate
+ *     (automatic, v0.8-A);
+ *   - a skill file is durable, discoverable and shared → evidence gate **plus**
+ *     an explicit human yes (this module).
+ *
+ * The record lives next to the ledger, not inside the memory store: an approval
+ * is a human act, and mixing it into the pool would let the same pool "approve"
+ * itself. Shape:
+ *
+ *   { "version": 1, "workspaces": { "<ws>": { "<memoryId>": { at, note } } } }
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+export const APPROVALS_VERSION = 1;
+
+export function approvalsPath(home) {
+  return path.join(home, "playbook-approvals.json");
+}
+
+/** Read the approval file; a missing or corrupt file means "nothing approved". */
+export function loadApprovals(home) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(approvalsPath(home), "utf8"));
+    if (parsed === null || typeof parsed !== "object") return empty();
+    return {
+      version: APPROVALS_VERSION,
+      workspaces: parsed.workspaces !== null && typeof parsed.workspaces === "object" ? parsed.workspaces : {},
+    };
+  } catch {
+    return empty();
+  }
+}
+
+function empty() {
+  return { version: APPROVALS_VERSION, workspaces: {} };
+}
+
+/** Approved ids for one workspace (empty when the workspace has none). */
+export function approvedIds(state, workspace) {
+  const bucket = state?.workspaces?.[workspace];
+  return bucket !== null && typeof bucket === "object" ? Object.keys(bucket) : [];
+}
+
+export function isApproved(state, workspace, memoryId) {
+  return state?.workspaces?.[workspace]?.[memoryId] !== undefined;
+}
+
+function save(home, state) {
+  const file = approvalsPath(home);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const tmp = `${file}.tmp`;
+  fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+  fs.renameSync(tmp, file);
+  return state;
+}
+
+export function recordApproval(home, workspace, memoryId, { note = null, at = Date.now() } = {}) {
+  if (typeof memoryId !== "string" || memoryId.length === 0) throw new Error("approve needs a memory id");
+  const state = loadApprovals(home);
+  state.workspaces[workspace] = state.workspaces[workspace] ?? {};
+  state.workspaces[workspace][memoryId] = { at, note };
+  return save(home, state);
+}
+
+export function revokeApproval(home, workspace, memoryId) {
+  const state = loadApprovals(home);
+  const bucket = state.workspaces[workspace];
+  if (bucket !== undefined) {
+    delete bucket[memoryId];
+    if (Object.keys(bucket).length === 0) delete state.workspaces[workspace];
+  }
+  return save(home, state);
+}

package/lib/boundary-prune.js

@@ -86,12 +86,34 @@ export function isBoundary(event, state) {
 }
 
 /** Sessions per workspace with ≥1 request, as (count, medianRequests) map — L0 repayment input. */
+/**
+ * One key for "which workspace is this", from either side of the lookup.
+ *
+ * Session buckets are named after the cwd with `/` flattened to `-`
+ * (`--Users-me-dsh-loom--`), while the live session header carries the absolute
+ * path (`/Users/me/dsh-loom`). Comparing those two strings never matches, and an
+ * unmatched workspace falls back to the bare floor — which makes the repayment
+ * gate (`requests + minRemaining > expectedTotal`) true for every session past
+ * its first request, i.e. L1 armed and permanently silent. Normalising both
+ * sides is what makes the history lookup real.
+ * @param {string | undefined | null} workspace
+ * @returns {string}
+ */
+export function normalizeWorkspaceKey(workspace) {
+  if (typeof workspace !== "string") return "";
+  const flat = workspace
+    .trim()
+    .replace(/[-\/\\]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return process.platform === "win32" ? flat.toLowerCase() : flat;
+}
+
 export function workspaceHistory(root = path.join(os.homedir(), ".dsh", "sessions")) {
   const byWorkspace = new Map();
   try {
     for (const bucket of fs.readdirSync(root)) {
       if (!bucket.startsWith("--")) continue;
-      const ws = decodeURIComponent(bucket.replace(/^--/, "").replace(/--$/, ""));
+      const ws = normalizeWorkspaceKey(decodeURIComponent(bucket.replace(/^--+/, "").replace(/--+$/, "")));
       let sessions = 0;
       const bdir = path.join(root, bucket);
       for (const sid of fs.readdirSync(bdir)) {
@@ -120,7 +142,7 @@ export function workspaceHistory(root = path.join(os.homedir(), ".dsh", "session
  * must not starve them).
  */
 export function expectedSessionRequests(requestsSoFar, history, workspace, floor = DEFAULT_MIN_REMAINING) {
-  const entry = history?.get(workspace);
+  const entry = history?.get(workspace) ?? history?.get(normalizeWorkspaceKey(workspace));
   const prior = entry && entry.sessions > 0 ? Math.max(floor, EMPIRICAL_MEDIAN_REQUESTS) : floor;
   if (requestsSoFar <= prior) return prior;
   // past the prior: project the session to run at least as long again as it
@@ -189,6 +211,15 @@ export function installBoundaryPrune(ctx, config, log, hooks = {}, opts = {}) {
   // this is purely evidence-based off, never speculative on.
   const adaptive = config?.boundaryPruneAdaptive !== false;
   const allowFire = typeof opts?.allowFire === "function" ? opts.allowFire : () => true;
+  // Diagnostics hook: every boundary the listener actually SEES is reported
+  // (skipped by the repayment gate, held by the ledger gate, or fired) so that
+  // "armed but never fires" can be told apart from "no boundary ever came".
+  // A throwing hook must never disturb the session pipeline.
+  const report = (kind, fields = {}) => {
+    try {
+      opts?.onAttempt?.({ kind, ...fields });
+    } catch { /* diagnostics only */ }
+  };
   const blockedNotified = new Set();
 
   // Optional threshold override (ResultPack L0, T2048 landing): the host pruner
@@ -258,7 +289,7 @@ export function installBoundaryPrune(ctx, config, log, hooks = {}, opts = {}) {
             const expectedTotal = expectedSessionRequests(state.requests, history, workspace, minRemaining);
             if (state.requests + minRemaining <= expectedTotal) {
               // reuse the same fire path via a synthetic boundary
-              return fireBoundary(session, { kind: "step" }, state, expectedTotal);
+              return fireDeferred(session, { kind: "step" }, state, expectedTotal);
             }
           }
         }
@@ -273,14 +304,40 @@ export function installBoundaryPrune(ctx, config, log, hooks = {}, opts = {}) {
       // requests beyond the current count.
       const workspace = wk(session?.header?.cwd);
       const expectedTotal = expectedSessionRequests(state.requests, history, workspace, minRemaining);
-      if (state.requests + minRemaining > expectedTotal) return;
-      return fireBoundary(session, boundary, state, expectedTotal);
+      if (state.requests + minRemaining > expectedTotal) {
+        report("skipped", { boundary, requests: state.requests, expectedTotal, workspace });
+        return;
+      }
+      return fireDeferred(session, boundary, state, expectedTotal);
     } catch (error) {
       // fully contained — a prune bug must never disturb the session pipeline
+      report("error", { where: "listener", message: String(error) });
       log?.warn?.(`engram boundary-prune threw: ${String(error)}`);
     }
   };
 
+  /**
+   * Fire OUTSIDE the `commit()` call stack.
+   *
+   * `session/event` observers run while the session is publishing an event
+   * (`commit()` sets `appending` for the duration). Pruning appends its own
+   * `compaction/prune` event, and DSH refuses a reentrant append ("session
+   * append cannot reenter while another append is being published"). Firing
+   * synchronously therefore works only while there is NOTHING to prune — the
+   * exact shape of "armed, reports fired, prunes zero, forever". The microtask
+   * runs after commit() returns, where append is legal again.
+   */
+  const fireDeferred = (session, boundary, state, expectedTotal) => {
+    queueMicrotask(() => {
+      try {
+        fireBoundary(session, boundary, state, expectedTotal);
+      } catch (error) {
+        report("error", { where: "fire", boundary, message: String(error) });
+        log?.warn?.(`engram boundary-prune: fire threw: ${String(error)}`);
+      }
+    });
+  };
+
   /** Shared fire path for every boundary source (event + step-cadence). */
   const fireBoundary = (session, boundary, state, expectedTotal) => {
     if (adaptive) {
@@ -292,6 +349,7 @@ export function installBoundaryPrune(ctx, config, log, hooks = {}, opts = {}) {
         allowed = true; // a gate bug must never disable pruning
       }
       if (!allowed) {
+        report("held", { boundary, requests: state.requests, workspace });
         if (!blockedNotified.has(workspace)) {
           blockedNotified.add(workspace);
           log?.info?.(
@@ -301,8 +359,16 @@ export function installBoundaryPrune(ctx, config, log, hooks = {}, opts = {}) {
         return;
       }
     }
+    const prunerWorkspace = wk(session?.header?.cwd);
     const result = activePruner.pruneSession(session);
     const prunedNow = result?.pruned?.length ?? 0;
+    report("fired", {
+      boundary,
+      requests: state.requests,
+      pruned: prunedNow,
+      charsRemoved: Number.isFinite(result?.charsRemoved) ? result.charsRemoved : 0,
+      workspace: prunerWorkspace,
+    });
     if (prunedNow > 0) {
       state.prunes += 1;
       state.prunedResults += prunedNow;

package/lib/capsule.js

@@ -0,0 +1,134 @@
+/**
+ * v0.8-C (advisory): subagent handoff capsules.
+ *
+ * The proposal wants the parent agent to receive a ≤500-token capsule instead of
+ * a child's full report ("conclusion / evidence / open questions / next step").
+ * That substitution needs a seam the host does not expose today: `tools/result`
+ * is emitted through `ctx.events.dispatch("emit", …)` and the emitter's own
+ * comment says it notifies "observers without exposing a mutation or error
+ * channel into the outcome" — a listener cannot rewrite what the parent reads.
+ *
+ * So this module ships the half that IS reachable, and is honest about the
+ * difference:
+ *
+ *   - it distills a child's report into a deterministic capsule and stores it as
+ *     a `handoff` memory, so the conclusion survives the child session and can be
+ *     recalled after a compaction (the full text stays re-fetchable in the child's
+ *     session log);
+ *   - it records the source/capsule sizes in the memory itself, so the
+ *     "how much would substitution have saved" question is measurable instead of
+ *     asserted.
+ *
+ * Zero LLM: the capsule is extracted by structure (headings, bullets, question
+ * marks), never summarized by a model.
+ */
+
+export const CAPSULE_TAG = "subagent-capsule";
+
+/** Tool names whose result is a subagent's final report. */
+export function isSubagentTool(name) {
+  return typeof name === "string" && (name === "subagent" || name === "subagent_fork" || name === "agent");
+}
+
+const HEADING_RE = /^\s{0,3}#{1,6}\s+(.*)$/;
+const BULLET_RE = /^\s*(?:[-*+]|\d+[.)])\s+(.*)$/;
+const EVIDENCE_RE = /(test|pass|fail|bench|artifact|eval|commit|✔|✓|✗|→|path|\/[\w.-]+\.(?:js|mjs|ts|md|json|yml|yaml|py|go|rs)\b|#\d|sha|https?:)/i;
+const OPEN_RE = /(\?|todo|open question|unresolved|blocked|未决|待定|风险|risk)/i;
+const NEXT_RE = /(next step|next:|下一步|建议|recommend|should (?:then|now)|follow-?up|remaining)/i;
+
+function cleanLine(line) {
+  return String(line ?? "")
+    .replace(/^[-*+]\s+/, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * Deterministic capsule of a long report.
+ *
+ * Sections are best-effort structural extraction; the footer always states what
+ * was compressed and where the original lives, so a reader can tell a capsule
+ * from a report.
+ */
+export function buildCapsule(text, { maxChars = 1800, source = "subagent report" } = {}) {
+  const full = String(text ?? "").trim();
+  if (full.length === 0) return { capsule: "", sourceChars: 0, capsuleChars: 0 };
+  // Keep raw lines: bullet/heading detection needs their prefixes, rendering does not.
+  const raw = full.split("\n").filter((l) => l.trim().length > 0);
+  const isHeading = (l) => /^\s{0,3}#{1,6}\s+/.test(l);
+  const bulletOf = (l) => /^\s*(?:[-*+]|\d+[.)])\s+(.*)$/.exec(l)?.[1] ?? null;
+
+  // Conclusion: the opening prose before the first heading/bullet, bounded so a
+  // wall of text cannot become the "conclusion".
+  const conclusionLines = [];
+  for (const line of raw) {
+    if (isHeading(line) || bulletOf(line) !== null) {
+      if (conclusionLines.length > 0) break;
+      continue;
+    }
+    conclusionLines.push(cleanLine(line));
+    if (conclusionLines.join(" ").length > 300) break;
+  }
+  const conclusion = conclusionLines.join(" ").slice(0, 300);
+
+  const bullets = raw.map(bulletOf).filter((b) => b !== null).map(cleanLine);
+  const evidence = bullets.filter((b) => EVIDENCE_RE.test(b)).slice(0, 5);
+  const open = bullets.filter((b) => OPEN_RE.test(b)).slice(0, 3);
+  const next = bullets.filter((b) => NEXT_RE.test(b) && !open.includes(b)).slice(0, 3);
+
+  const parts = [];
+  const structured = evidence.length > 0 || open.length > 0 || next.length > 0;
+  if (structured) {
+    if (conclusion.length > 0) parts.push(`**结论**：${conclusion}`);
+    if (evidence.length > 0) parts.push(`**证据**：\n${evidence.map((e) => `- ${e}`).join("\n")}`);
+    if (open.length > 0) parts.push(`**未决**：\n${open.map((e) => `- ${e}`).join("\n")}`);
+    if (next.length > 0) parts.push(`**下一步**：\n${next.map((e) => `- ${e}`).join("\n")}`);
+  } else {
+    // Unstructured prose: keep both ends and mark the elision instead of
+    // pretending a summary happened.
+    const head = raw.slice(0, 2).map(cleanLine).join("\n");
+    const tail = raw.slice(-2).map(cleanLine).join("\n");
+    parts.push(`**无结构可提炼（保留首尾）**：\n${head}\n…\n${tail}`);
+  }
+
+  const footer = (chars) =>
+    `\n\n---\n胶囊：${source} ${full.length} → ${chars} 字符；全文在子代理会话日志里（可用 session 搜索回取）。`;
+  const room = Math.max(200, maxChars - 120);
+  let body = parts.join("\n\n");
+  if (body.length > room) body = `${body.slice(0, room).trimEnd()}…`;
+
+  let capsule = `${body}${footer(body.length)}`;
+  // A "compression" that is not smaller is a bug, not a summary: fall back to a
+  // straight head/tail cut that is guaranteed to fit.
+  if (capsule.length >= full.length) {
+    const cut = Math.max(120, Math.floor((full.length - 200) * 0.7));
+    const head = full.slice(0, cut).trimEnd();
+    const tail = full.slice(-Math.min(80, Math.max(0, full.length - cut - 1))).trim();
+    body = `**无结构可提炼（保留首尾）**：\n${head}…\n…${tail}`;
+    capsule = `${body}${footer(body.length)}`;
+    if (capsule.length >= full.length) capsule = full.slice(0, Math.max(80, full.length - 1));
+  }
+  return { capsule, sourceChars: full.length, capsuleChars: capsule.length };
+}
+
+/** Read the text out of a tool result (string, content blocks, or value fields). */
+export function resultText(result) {
+  const value = result?.value;
+  if (typeof value === "string" && value.length > 0) return value;
+  if (value !== null && typeof value === "object") {
+    if (Array.isArray(value.content)) {
+      const joined = value.content
+        .filter((b) => b?.type === "text" && typeof b.text === "string")
+        .map((b) => b.text)
+        .join("\n");
+      if (joined.length > 0) return joined;
+    }
+    if (typeof value.text === "string" && value.text.length > 0) return value.text;
+  }
+  const content = result?.content;
+  if (Array.isArray(content)) {
+    const joined = content.filter((b) => b?.type === "text" && typeof b.text === "string").map((b) => b.text).join("\n");
+    if (joined.length > 0) return joined;
+  }
+  return null;
+}

package/lib/capture.js

@@ -21,6 +21,8 @@
  */
 
 import { workspaceKey as workspaceKeyOf } from "./util.js";
+import { sessionSeq } from "./session-events.js";
+import { buildCapsule, isSubagentTool, resultText } from "./capsule.js";
 
 const GIT_MILESTONE_SIGNAL = 0.5;
 const GIT_COMMIT_SIGNAL = 0.55;
@@ -50,7 +52,7 @@ export function cmdTag(command) {
  * error rows. Deterministic; fires only on genuine same-command
  * success-after-failure.
  */
-export function fixClosureEntry(name, command, result, previousErrors) {
+export function fixClosureEntry(name, command, result, previousErrors, at = Date.now()) {
   if (!["bash", "shell", "exec"].includes(name)) return null;
   if (result?.isError === true) return null; // not a success
   if (typeof command !== "string" || command.trim().length === 0) return null;
@@ -70,11 +72,38 @@ export function fixClosureEntry(name, command, result, previousErrors) {
       signal: FIX_CLOSURE_SIGNAL,
       tags: ["procedure", "auto-captured", "fix", tag, ...(isTest ? ["test"] : [])],
       filePath: null,
+      // One success is never enough to generalize (v0.8-A): the row starts with
+      // a single evidence point and has to earn an index line.
+      evidenceCount: 1,
+      lastVerifiedAt: at,
     },
     resolvedIds: failing.map((m) => m.id),
   };
 }
 
+/**
+ * v0.8-A: what should happen to an existing `fixed:` procedure when the SAME
+ * command runs again? Returns `{ procedureId, ok }` or null.
+ *
+ * Success → credit one evidence point; failure → invalidate immediately.
+ * Deterministic, zero LLM, and scoped to the workspace's active rows.
+ */
+export function procedureOutcome(name, command, result, rows) {
+  if (!isCommandRun(name)) return null;
+  if (typeof command !== "string" || command.trim().length === 0) return null;
+  const tag = cmdTag(command);
+  if (tag === null) return null;
+  const procedure = (rows ?? []).find(
+    (m) =>
+      m?.kind === "procedure" &&
+      m.status === "active" &&
+      /^fixed:/.test(String(m.text ?? "")) &&
+      (m.tags ?? []).includes(tag),
+  );
+  if (procedure === void 0) return null;
+  return { procedureId: procedure.id, ok: result?.isError !== true };
+}
+
 /** Git operations that carry lasting meaning (milestones). */
 const GIT_OPS = [
   // commit: only with an explicit -m subject — that's the semantic content.
@@ -153,9 +182,42 @@ export function makeCaptureHandler(store, config, log, stats) {
 
       const args = (exec.arguments ?? {});
       const git = extractGit(name, args.command);
+      // Evidence accounting applies to EVERY command run, not only the runs that
+      // close a failure: a procedure only keeps its index line while it keeps
+      // holding (v0.8-A).
+      const recentRows = store.listMemories !== void 0 ? store.listMemories(workspaceKeyOf(cwd), 200) : [];
+      const outcome = procedureOutcome(name, args.command, result, recentRows);
+      const workspace = workspaceKeyOf(cwd);
+      const applyOutcome = (errorId = null) => {
+        if (outcome === null) return;
+        void store
+          .recordProcedureOutcome?.(workspace, outcome.procedureId, { ok: outcome.ok, errorId })
+          ?.catch?.(() => null);
+      };
       let entry = null;
 
-      if (git !== null) {
+      // v0.8-C (advisory): a subagent's report is durable knowledge worth
+      // keeping even though the host will not let us shrink what the parent
+      // reads (see lib/capsule.js). Long reports become a capsule memory.
+      const subagentReport = isSubagentTool(name) && result?.isError !== true ? resultText(result) : null;
+      const capsuleMin = config.capsuleMinSourceChars ?? 1500;
+      if (subagentReport !== null && subagentReport.length >= capsuleMin) {
+        const { capsule } = buildCapsule(subagentReport, {
+          maxChars: config.capsuleMaxChars ?? 1800,
+          source: String(args.description ?? name),
+        });
+        entry = {
+          kind: "handoff",
+          text: capsule,
+          signal: 0.55,
+          tags: ["handoff", "auto-captured", "subagent-capsule", `agent:${String(args.description ?? name).slice(0, 60)}`],
+          filePath: null,
+        };
+      }
+
+      if (entry !== null) {
+        // already drafted above
+      } else if (git !== null) {
         entry = { kind: "decision", text: git.text, signal: git.signal, tags: git.tags };
       } else if (result?.isError === true) {
         // Errors take precedence over tool-name classification: a failed edit
@@ -198,7 +260,13 @@ export function makeCaptureHandler(store, config, log, stats) {
         }
       }
 
-      if (entry === null) return;
+      if (entry === null) {
+        // No new memory to store, but a command run still carries evidence for
+        // an existing procedure (v0.8-A): credit the success, or invalidate on
+        // failure. This is the common case — most runs produce no memory at all.
+        applyOutcome();
+        return;
+      }
       if (stats) {
         stats.total += 1;
         if (entry.kind === "decision") stats.git += 1;
@@ -208,7 +276,14 @@ export function makeCaptureHandler(store, config, log, stats) {
 
       // exact-duplicate guard across the recent window (store re-dedups by hash anyway)
       const key = `${entry.kind}|${entry.text}`;
-      if (recent.has(key)) return;
+      if (recent.has(key)) {
+        // Exact repeat within the window: nothing new to store, but the run is
+        // still evidence for or against the procedure (v0.8-A). A repeated
+        // failure invalidates by the row it repeats, not by a phantom id.
+        const existing = recentRows.find((m) => m.kind === entry.kind && (m.text ?? "") === entry.text);
+        applyOutcome(existing?.id ?? null);
+        return;
+      }
       recent.add(key);
       if (recent.size > 500) {
         const first = recent.values().next().value;
@@ -225,17 +300,21 @@ export function makeCaptureHandler(store, config, log, stats) {
             tags: entry.tags,
             signal: entry.signal,
             filePath: entry.filePath ?? null,
+            // v0.8-A: a closure row carries its first evidence point at birth.
+            evidenceCount: entry.evidenceCount ?? 0,
+            lastVerifiedAt: entry.lastVerifiedAt ?? null,
             sessionId,
-            seq: agent.session.events?.length ?? 0,
+            seq: sessionSeq(agent.session),
           },
           config,
         )
-        .then(() => {
+        .then((saved) => {
           // Failure→fix closure: mark the earlier error rows resolved (tag
           // append only — never re-ranks them, per store.tagMemory's contract).
           for (const id of resolvedIds) {
             void store.tagMemory?.(workspaceKeyOf(cwd), id, "resolved")?.catch?.(() => null);
           }
+          applyOutcome(saved?.id ?? null);
         })
         .catch((error) => log?.warn?.(`engram auto-capture failed: ${String(error)}`));
     } catch (error) {