dsh-engram 0.7.4 → 0.8.6

package/lib/context-gc.js

@@ -33,6 +33,7 @@
 
 import { basename } from "node:path";
 import { slugId } from "./util.js";
+import { recordGcAttempt } from "./status.js";
 
 /** Memory ids are uuids; the [ENGRAM] index shows `#` + first 8 alnum chars. */
 const MEMORY_ID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
@@ -608,6 +609,43 @@ export async function summarizeUnprovenanced(messages, { enabled = true, maxChar
  * Build the Context GC replacement summary (pointer block + narrative). This
  * is the text that lands inside DSH's `<compacted-summary>` checkpoint node.
  */
+/** The line every Context GC checkpoint carries — proof of authorship when a
+ *  committed summary is read back from the session log. */
+export const CONTEXT_GC_HEADING = "## Context GC — evicted detail is RE-FETCHABLE (do not re-derive)";
+export const CONTEXT_GC_FOOTER = "_Context GC: detail above is re-fetchable via the listed tools. Prefer re-fetching over re-deriving._";
+
+/**
+ * Turn a committed `compaction/summary` event into a Context Ledger credit, or
+ * `null` when engram did not author it.
+ *
+ * The credit has to come from the COMMITTED record, not from inside
+ * `summarize()`: the harness may reject a checkpoint (framed ≥ shadowed) or fall
+ * back to its own summarizer, and a saving that never reached the session is not
+ * a saving. `shadowedTokenCount` is the span that stopped replaying, the summary
+ * text is what replaced it.
+ */
+export function committedGcSaving(event, charsPerToken = 4) {
+  if (event?.type !== "compaction/summary") return null;
+  // DSH publishes committed events as { type, seq, time, data: { summary,
+  // shadowedTokenCount, … } }. Reading the payload off the event root made this
+  // matcher return null for EVERY real checkpoint — 82 GC-authored summaries on
+  // this machine, none credited — while the unit test kept passing because it
+  // built a flat synthetic event. Accept both shapes; prefer the real one.
+  const payload = event?.data !== null && typeof event?.data === "object" ? event.data : event;
+  const text = Array.isArray(payload?.summary)
+    ? payload.summary.map((part) => (typeof part?.text === "string" ? part.text : "")).join("")
+    : "";
+  // Either end identifies the checkpoint: the heading survives a trim (it is
+  // the head), the footer is the cheaper full-text check. A checkpoint trimmed
+  // to fit keeps the heading but may lose the footer, and it still committed.
+  const authored = text.includes(CONTEXT_GC_FOOTER) || [...text].slice(0, 200).join("").includes(CONTEXT_GC_HEADING);
+  if (!authored) return null;
+  const shadowed = Number(payload?.shadowedTokenCount ?? event?.shadowedTokenCount);
+  if (!Number.isFinite(shadowed) || shadowed <= 0) return null;
+  const charsSaved = shadowed * charsPerToken - [...text].length;
+  return charsSaved > 0 ? { charsSaved, shadowedTokens: shadowed, summaryChars: [...text].length } : null;
+}
+
 export function buildContextGcSummary(report, context, { turns, narrative }) {
   const label = context?.label ?? "this workspace";
   const pointer = buildPointerSummary(report, context, { turns, label });
@@ -617,7 +655,7 @@ export function buildContextGcSummary(report, context, { turns, narrative }) {
     narrative.text,
     "",
     "---",
-    "_Context GC: detail above is re-fetchable via the listed tools. Prefer re-fetching over re-deriving._",
+    CONTEXT_GC_FOOTER,
   ];
   return parts.join("\n");
 }
@@ -732,15 +770,37 @@ export async function loadContextGcEngine(ctx, config = {}, deps = {}) {
     constructor() {
       super(ctx, engineConfig);
       this.gcDeps = deps;
+      /** Last credit computed by `_contextGc`, surfaced in the attempt log. */
+      this.gcLastCredit = void 0;
     }
 
     /** The one seam we take over: produce the checkpoint summary. */
     async summarize(input, agent, signal) {
+      const startedAt = Date.now();
+      const workspace = agent?.session?.header?.cwd;
       try {
-        return await this._contextGc(input, agent, signal);
+        const result = await this._contextGc(input, agent, signal);
+        // Evidence, not vibes: the ledger's credit side only moves when THIS
+        // path runs, so every attempt is recorded where it can be inspected
+        // offline (a silent fallback used to look identical to a working GC).
+        // The head + length identify the checkpoint: if the committed
+        // `compaction/summary` in the session log does not start like this, the
+        // harness used someone else's summary and nothing was saved.
+        const text = Array.isArray(result?.summary) ? result.summary.map((b) => (typeof b?.text === "string" ? b.text : "")).join("") : "";
+        await recordGcAttempt({
+          plane: "compaction",
+          ok: true,
+          ms: Date.now() - startedAt,
+          workspace,
+          charsSaved: this.gcLastCredit?.charsSaved ?? 0,
+          summaryChars: [...text].length,
+          head: [...text].slice(0, 60).join(""),
+        });
+        return result;
       } catch (error) {
         const logger = this.ctx?.logger;
         const message = error instanceof Error ? error.message : String(error);
+        await recordGcAttempt({ plane: "compaction", ok: false, ms: Date.now() - startedAt, workspace, error: message });
         if (typeof logger?.warn === "function") logger.warn(`engram context-gc failed (${message}); falling back to default compaction`);
         if (typeof this.gcDeps.defaultSummarize === "function") {
           return this.gcDeps.defaultSummarize(input, agent, signal);
@@ -804,6 +864,7 @@ export async function loadContextGcEngine(ctx, config = {}, deps = {}) {
           const summaryChars = [...summaryText].length;
           const charsSaved = spanChars - summaryChars;
           if (charsSaved > 0) {
+            this.gcLastCredit = { charsSaved, spanChars, summaryChars };
             this.gcDeps.onCompacted({ session: agent?.session, workspace: ws, charsSaved, spanChars, summaryChars });
           }
         }

package/lib/index-block.js

@@ -13,6 +13,7 @@ import { label, shortId, fmtDate, escapeLt, truncate, byRecency } from "./util.j
 import { buildEntityPage, renderEntityCard } from "./entity-page.js";
 import { memoryIsStale } from "./anchor.js";
 import { readSessionEvents } from "./session-events.js";
+import fs from "node:fs";
 
 /** Deterministic index line prefix per memory kind. */
 export function prefixFor(kind) {
@@ -42,9 +43,77 @@ function isGitPlumbingEcho(m) {
   return /(^|\s)(cd\s|\&\&|;|\|)/.test(text);
 }
 
-/** A procedure that has been used (hit) enough times to count as proven. */
+/**
+ * Has the file this memory is anchored to changed since the memory was written?
+ *
+ * The reverse G6 experiment (docs/G6-quality-ab.zh.md §9) is why this exists:
+ * with a stale convention in memory and a contradicting docstring in the code,
+ * the agent followed memory on 1 of 6 tasks because nothing marked the memory as
+ * old. A memory that names a file can carry that check for free — one `stat`,
+ * and the injected line can say "the file moved on".
+ *
+ * Returns "" when there is nothing to say (no anchor, file unchanged, or the
+ * stat fails); a short marker otherwise. Never throws: injection must not break
+ * because a path is unreadable.
+ */
+export function fileDrift(m, nowMs = Date.now(), stat = void 0) {
+  const file = typeof m?.filePath === "string" ? m.filePath : "";
+  if (file.length === 0) return "";
+  const written = m?.createdAt ?? m?.updatedAt;
+  if (written === null || written === void 0) return "";
+  const writtenMs = typeof written === "number" ? written : Date.parse(String(written));
+  if (!Number.isFinite(writtenMs) || writtenMs > nowMs) return "";
+  const doStat = stat ?? ((target) => fs.statSync(target));
+  let statInfo;
+  try {
+    statInfo = doStat(file);
+  } catch {
+    return "⚠file-missing";
+  }
+  const mtimeMs = statInfo?.mtimeMs ?? 0;
+  if (!Number.isFinite(mtimeMs) || mtimeMs <= writtenMs) return "";
+  // Only a coarse signal: "the anchored file was edited after this was written".
+  // The model can then check the file instead of trusting the note.
+  return "⚠file-changed";
+}
+
+/**
+ * v0.8-A promotion rules for procedures.
+ *
+ * A procedure earns an index line only when it is still true:
+ *   - verified at least `procedureMinEvidence` (3) times,
+ *   - verified within `procedureMaxAgeDays` (14),
+ *   - and not failed since its last verification.
+ * A manual procedure with no evidence fields keeps the older hit-based path
+ * (`hits >= promoteHits`), so hand-written procedures are not silently demoted.
+ */
+export function procedureInvalidated(m) {
+  if ((m?.kind ?? "") !== "procedure") return false;
+  if (m?.lastFailedAt === null || m?.lastFailedAt === void 0) return false;
+  return m.lastVerifiedAt === null || m.lastVerifiedAt === void 0 || m.lastFailedAt >= m.lastVerifiedAt;
+}
+
 export function isProvenProcedure(m, config) {
-  return (m?.kind ?? "") === "procedure" && (m?.hits ?? 0) >= (config?.promoteHits ?? 3);
+  if ((m?.kind ?? "") !== "procedure") return false;
+  if (procedureInvalidated(m)) return false;
+  const nowMs = config?.nowMs ?? Date.now();
+  const minEvidence = config?.procedureMinEvidence ?? 3;
+  const maxAgeMs = (config?.procedureMaxAgeDays ?? 14) * 24 * 60 * 60 * 1000;
+  if ((m.evidenceCount ?? 0) > 0) {
+    const verified = m.lastVerifiedAt ?? 0;
+    return (m.evidenceCount ?? 0) >= minEvidence && verified > 0 && nowMs - verified <= maxAgeMs;
+  }
+  return (m?.hits ?? 0) >= (config?.promoteHits ?? 3);
+}
+
+/**
+ * An auto-captured `fixed:` row is a hypothesis until it has evidence — it
+ * must not earn an index line on the strength of one success.
+ */
+export function isUnprovenAutoProcedure(m, config) {
+  if ((m?.kind ?? "") !== "procedure") return false;
+  if (!Array.isArray(m.tags) || !m.tags.includes("auto-captured")) return false;
+  return !isProvenProcedure(m, config);
 }
 
 /**
@@ -67,6 +136,10 @@ export function selectIndexMemories(store, workspace, config) {
     // A memory whose file source changed/vanished is no longer current truth
     // for an auto-injected index line (still re-fetchable via engram_detail).
     .filter((m) => !memoryIsStale(m))
+    // v0.8-A: a just-invalidated procedure is worse than none — the next agent
+    // would copy it verbatim. Unproven auto-captures stay out until they earn it.
+    .filter((m) => !procedureInvalidated(m))
+    .filter((m) => !isUnprovenAutoProcedure(m, config))
     .filter(
       (m) =>
         (isGitPlumbingEcho(m) && (m.hits ?? 0) < config.promoteHits
@@ -99,7 +172,8 @@ function memoryLine(m, config) {
   const mark = isProvenProcedure(m, config) ? `${px}✓` : px;
   const date = fmtDate(m.createdAt ?? m.updatedAt);
   const text = truncate(String(m.text).replace(/\s*\n+\s*/g, " ").trim(), INDEX_LINE_CHARS);
-  return `[${mark}] ${date} ${escapeLt(text)} #${shortId(m.id)}`;
+  const drift = fileDrift(m, config?.nowMs ?? Date.now(), config?.stat);
+  return `[${mark}] ${date} ${escapeLt(text)}${drift === "" ? "" : ` ${drift}`} #${shortId(m.id)}`;
 }
 
 function taskStateLine(t) {
@@ -274,7 +348,8 @@ export function renderStartRecall(store, workspace, config = {}, opts = {}) {
     const truncated = [...full].length > 140;
     const text = truncate(full, 140);
     if (truncated && truncatedId === null) truncatedId = shortId(m.id);
-    lines.push(`- [${mark}] ${date} ${escapeLt(text)} #${shortId(m.id)}${truncated ? " …" : ""}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
+    const drift = fileDrift(m, config?.nowMs ?? Date.now(), config?.stat);
+    lines.push(`- [${mark}] ${date} ${escapeLt(text)}${drift === "" ? "" : ` ${drift}`} #${shortId(m.id)}${truncated ? " …" : ""}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
   }
   const drill = truncatedId !== null
     ? `context: use above · #${truncatedId} truncated — engram_detail ${truncatedId} (full) · engram_recall <query>`

package/lib/index.js

@@ -27,19 +27,19 @@ import { installPreferenceCapture } from "./preference.js";
 import { boundaryPruneAvailability, installBoundaryPrune } from "./boundary-prune.js";
 import { resultPackDir, archivePrunedResults, resultRecallTool, sweepExpired } from "./result-pack.js";
 import { recordBoundaryPruneFire, recordRecallInjection, readPackTelemetry, engramHome } from "./pack-telemetry.js";
-import { createLedger, estimatePrefixChars } from "./ledger.js";
+import { CHARS_PER_TOKEN, createLedger, estimatePrefixChars } from "./ledger.js";
 import { createJit } from "./jit.js";
 import { buildEpisode } from "./episode.js";
 import { collectStaleAnchors } from "./anchor.js";
 import { renderIndex, renderStartRecall, renderResumeBrief, firstUserMessageText, renderEsr, esrMethodology, buildEsrSnapshot, buildFrozenEsrBlock } from "./index-block.js";
-import { sessionEventsSource, sessionSeq } from "./session-events.js";
+import { describeSessionSurface, sessionEventsSource, sessionSeq } from "./session-events.js";
 import { makeTriggerRecorder } from "./trigger.js";
 import { pendingTodosFromSession, sinkPendingTodos } from "./todo-sink.js";
 import { installEngramSettings, makeEngineConfigSchema, SETTINGS_KEYS } from "./settings.js";
 import { makeEngramRoutes } from "./api.js";
-import { collectPointerContext, mountCompactionEngine } from "./context-gc.js";
+import { collectPointerContext, committedGcSaving, mountCompactionEngine } from "./context-gc.js";
 import { provisionWebCompaction } from "./web-provision.js";
-import { writeContextGcStatus } from "./status.js";
+import { writeContextGcStatus, recordPruneAttempt } from "./status.js";
 import { workspaceKey as wk } from "./util.js";
 
 /** Stable cordis plugin name. */
@@ -81,11 +81,23 @@ const DEFAULTS = {
   // memories that outrank the [ENGRAM] index.
   autoCapturePreferences: true,
   preferenceCapturePerSession: 8,
+  // v0.8-C (advisory): a subagent report above this many chars is distilled
+  // into a durable `handoff` capsule. The parent still receives the full text —
+  // DSH's tools/result is observer-only (no mutation channel) — so this buys
+  // durability and a measurable saving, not a smaller prompt. See lib/capsule.js.
+  capsuleMinSourceChars: 1500,
+  capsuleMaxChars: 1800,
   // [ENGRAM] index budget (token discipline)
   indexMaxLines: 12,
   indexMaxChars: 700,
   minIndexSignal: 0.4,
   promoteHits: 3,
+  // v0.8-A evidence-based procedure promotion: a `fixed:` procedure reaches the
+  // [ENGRAM] index only after this many verified successes, and only while the
+  // last verification is younger than procedureMaxAgeDays. A later failure
+  // invalidates it immediately (see store.recordProcedureOutcome).
+  procedureMinEvidence: 3,
+  procedureMaxAgeDays: 14,
   // retention / caps
   expireDays: 180,
   maxMemoriesPerWorkspace: 2000,
@@ -186,9 +198,13 @@ const DEFAULTS = {
   // (deterministic surface-replace, zero LLM). L0 simulation (77 sessions):
   // todo-progress boundaries cover 79% of replay exposure; the repayment
   // gate below lifted cache-aware net savings 12–32% on every threshold.
-  // Default ON since v0.6 (PROPOSAL-evolution §3.5): `false` is the opt-out;
-  // the pruner service is feature-detected — absent ⇒ disabled with one log.
-  boundaryPrune: true,
+  //
+  // Default OFF since v0.7.11: 15 h of live accounting showed the host prunes
+  // over-threshold results by itself (64,200 chars across two sessions with
+  // zero engram involvement), while the boundary source never fired in an
+  // ordinary 220-request session — it only pays for itself in planning
+  // sessions that actually use todos/goals. `true` is the opt-in.
+  boundaryPrune: false,
   // Adaptive off-switch (v0.6-B): the Context Ledger vetoes further fires in a
   // workspace whose own accounting is net negative after real fires. Evidence
   // can only turn pruning OFF — never on. `false` = repayment gate only.
@@ -422,6 +438,8 @@ export function apply(ctx, config) {
 
   const service = {
     config: resolved,
+    /** Where local (non-store) engram state lives — approvals, telemetry, ledger. */
+    engramHome: () => engramHome(),
     getDomain,
     openedDomain: () => opened,
     recorder,
@@ -574,7 +592,7 @@ export function apply(ctx, config) {
           if (!pendingRecall && !warnedUnreadableLog) {
             warnedUnreadableLog = true;
             log?.warn?.(
-              "engram: this host exposes no readable session event log (no session.events, no ctx.sessionQuery, no snapshotEvents) — [RECALL] / episode / todo-sink stay dormant. Reporting once per session.",
+              `engram: this host exposes no readable session event log (no session.events, no ctx.sessionQuery, no snapshotEvents, no eventAt) — [RECALL] / episode / todo-sink stay dormant. Reporting once per session. Session surface: ${describeSessionSurface(session).join(", ") || "(empty)"}`,
             );
           }
         } else {
@@ -668,6 +686,9 @@ export function apply(ctx, config) {
 
   ctx.effect(() => {
     const disposers = [];
+    // sessionId → { seqs, at }: the prunes this plugin caused, so the
+    // compaction/prune observer can tell engram's savings from the host's.
+    const ownPruneSeqs = new Map();
 
     // Settings card edits (volatile Config fields) hot-commit into the refs
     // in place; re-read them so `resolved` reflects new values for new
@@ -703,16 +724,10 @@ export function apply(ctx, config) {
           return null;
         }
       },
-      // Context Ledger credit: the Context GC engine measures the span it
-      // replaced with a checkpoint (`span − summary` chars) and reports it
-      // here; the ledger stops the "savings are always 0" asymmetry.
-      onCompacted: ({ session, workspace, charsSaved }) => {
-        try {
-          ledger?.noteCompaction(session?.id, workspace ?? wk(session?.header?.cwd), { charsSaved });
-        } catch (error) {
-          log?.warn?.(`engram ledger compaction credit failed: ${String(error)}`);
-        }
-      },
+      // Credit is committed-record-driven (see the `compaction/summary`
+      // listener below), not summarize()-driven: a checkpoint the harness
+      // rejects or overrides saves nothing, and crediting it would make the
+      // ledger claim savings that never landed.
     });
     disposers.push(() => {
       // The engine's service registration is fiber-scoped and auto-removed on
@@ -735,6 +750,7 @@ export function apply(ctx, config) {
       if (!hostSettled || !webSettled) return;
       if (statusTimer !== void 0) { clearTimeout(statusTimer); statusTimer = void 0; }
       void writeContextGcStatus({
+        pid: process.pid,
         planes: resolved.autoWebCompaction === false ? ["host"] : ["host", "web"],
         host: service.compactionStatus.host,
         web: service.compactionStatus.web,
@@ -886,6 +902,45 @@ export function apply(ctx, config) {
         ledger.dispose(session?.id);
       });
       disposers.push(disposeLedgerSession);
+      // Compaction credit, committed-record-driven: only a summary that carries
+      // the Context GC footer AND actually landed in the session counts. The
+      // engine's own measurement (`summarize`) is not evidence — the harness can
+      // reject or override it, and then nothing was saved.
+      const disposeLedgerCompaction = ctx.on("session/event", (session, event) => {
+        const credit = committedGcSaving(event);
+        if (credit === null) return;
+        ledger.noteCompaction(session?.id, wk(session?.header?.cwd), { charsSaved: credit.charsSaved });
+      });
+      disposers.push(disposeLedgerCompaction);
+      // Prune accounting: DSH prunes over-threshold tool results by itself
+      // (mid-step, no boundary involved). Those savings are real but they are
+      // NOT engram's, so they land in `hostPrunedChars` — a visible leg that
+      // never feeds NetTokens. Prunes we caused are already credited by the
+      // boundary hook (matched by seq here and skipped).
+      const disposeLedgerPrune = ctx.on("session/event", (session, event) => {
+        if (event?.type !== "compaction/prune") return;
+        const sid = session?.id;
+        const workspace = wk(session?.header?.cwd);
+        const tokens = Number(event?.data?.shadowedTokenCount);
+        if (!Number.isFinite(tokens) || tokens <= 0) return;
+        const shadowed = Array.isArray(event?.data?.shadowedSeqs) ? event.data.shadowedSeqs : [];
+        const own = typeof sid === "string" ? ownPruneSeqs.get(sid) : void 0;
+        let source = "host";
+        if (own !== void 0) {
+          const ours = shadowed.some((seq) => own.seqs.has(seq));
+          // A host prune that merely follows ours within the same step still
+          // counts as host work; only overlapping seqs mean "we did this".
+          if (ours) source = "engram";
+          if (Date.now() - own.at > 120_000) ownPruneSeqs.delete(sid);
+        }
+        if (source === "engram") return;
+        ledger.notePrune(sid, workspace, {
+          charsRemoved: tokens * CHARS_PER_TOKEN,
+          rewriteChars: estimatePrefixChars(session) + tokens * CHARS_PER_TOKEN,
+          source: "host",
+        });
+      });
+      disposers.push(disposeLedgerPrune);
     }
 
     // JIT observers: tool results (error recurrence / file focus) and goal
@@ -1052,6 +1107,14 @@ export function apply(ctx, config) {
           // surface is already rewritten when this hook runs).
           const removed = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
           const prefixChars = estimatePrefixChars(session);
+          if (typeof session?.id === "string") {
+            const seqs = new Set();
+            for (const item of report?.pruned ?? []) {
+              const seq = item?.seq ?? item?.event?.seq ?? item?.payload?.seq;
+              if (Number.isFinite(seq)) seqs.add(seq);
+            }
+            ownPruneSeqs.set(session.id, { seqs, at: Date.now() });
+          }
           ledger?.notePrune(session?.id, workspace, {
             charsRemoved: removed,
             rewriteChars: prefixChars + removed,
@@ -1077,12 +1140,22 @@ export function apply(ctx, config) {
         ctx.inject(["toolResultPruner"], (prunerCtx) => {
           const dispose = installBoundaryPrune(prunerCtx, resolved, log, boundaryHooks, {
             allowFire: boundaryGate,
+            // Durability for "armed but never fires": this plane sees the host's
+            // own sessions; the agent-preset realm reports the same shape under
+            // plane "preset" (lib/prune.js). Comparing the two tells us where a
+            // missing boundary went.
+            onAttempt: (attempt) => {
+              void recordPruneAttempt({ plane: "host", ...attempt });
+            },
           });
           // L1 provably can (or cannot) fire from here on: record it for the
           // durable snapshot, then rewrite it — the first flush may have landed
           // before the pruner activated, and this status line is what users
           // read when "节省" stays at 0.
           boundaryPruneState = boundaryPruneAvailability(prunerCtx, resolved);
+          // Mirror it onto the live view so `/overview` can answer "is the only
+          // ledger credit path actually able to fire?" without reading the file.
+          service.compactionStatus.boundaryPrune = boundaryPruneState;
           flushStatus();
           return dispose;
         });