dsh-engram 0.5.0 → 0.6.0

package/lib/index.js

@@ -27,6 +27,8 @@ import { installPreferenceCapture } from "./preference.js";
 import { installBoundaryPrune } from "./boundary-prune.js";
 import { resultPackDir, archivePrunedResults, resultRecallTool, sweepExpired } from "./result-pack.js";
 import { recordBoundaryPruneFire, recordRecallInjection, readPackTelemetry, engramHome } from "./pack-telemetry.js";
+import { createLedger, estimatePrefixChars } from "./ledger.js";
+import { createJit } from "./jit.js";
 import { renderIndex, renderStartRecall, firstUserMessageText, renderEsr, esrMethodology, buildEsrSnapshot, buildFrozenEsrBlock } from "./index-block.js";
 import { makeTriggerRecorder } from "./trigger.js";
 import { pendingTodosFromSession, sinkPendingTodos } from "./todo-sink.js";
@@ -92,9 +94,11 @@ const DEFAULTS = {
   gcStableRetentionDays: 120,
   // Context GC — the auto-GC that REPLACES DSH's built-in lossy LLM-summary
   // compaction (pi-esr CONTEXT_GC): mechanical eviction + re-fetch pointers,
-  // NOT the memory-panel GC above. `gcReplacesCompaction` off → the host keeps
-  // its default compaction backend; `gcNarrative` off → fully mechanical.
-  gcReplacesCompaction: true,
+  // NOT the memory-panel GC above. Opt-in since v0.6: rewriting host presets is
+  // the highest-maintenance action in the plugin (DSH broke the settings API
+  // twice), so the default is the host summarizer; `gcNarrative` off → fully
+  // mechanical. Turn this on to replace compaction with pointer summaries.
+  gcReplacesCompaction: false,
   gcNarrative: true,
   gcNarrativeMaxTokens: 1024,
   gcNarrativeMaxChars: 4000,
@@ -109,6 +113,21 @@ const DEFAULTS = {
   engramIndexOrder: 40,
   esrMethodOrder: 40.5, // static ESR protocol section (byte-identical every turn)
   esrOrder: 41,
+  // Context Ledger (PROPOSAL-evolution §4 v0.6-A): per-session accounting of
+  // injected block chars, pruned/compacted savings and prefix-rewrite penalty.
+  // In-memory + daily rollup at ~/.dsh/engram/ledger.json; never on the hot
+  // path of model calls (one integer add per request). `ledgerCacheDiscount`
+  // is the cached-prefix price share (0.1 = cache hit costs 10%, so a rewrite
+  // costs 90% of the prefix).
+  ledgerEnabled: true,
+  ledgerCacheDiscount: 0.1,
+  // JIT push (v0.6-C): append at most a few one-line, high-precision hints at
+  // deterministic events (error recurrence, file focus, goal completion).
+  // Append-only; every push pays a prefix rewrite in the ledger.
+  jitEnabled: true,
+  jitMaxLines: 3,
+  jitMaxLineChars: 220,
+  engramJitOrder: 40.75,
   // decision-point triggers (P0-A): promote a session plan to an ESR task
   // when the pending-todo count is at least minTodosForPromote and exceeds
   // active ESR tasks × todoPromoteRatio (default: pending > active).
@@ -134,7 +153,8 @@ const DEFAULTS = {
   // validation + idempotent; a preset with a custom compaction layout — or
   // none, like `minimal` — is never touched). Off = every preset keeps DSH's
   // default LLM summarizer; the host plane (headless/TUI/base) is unaffected.
-  autoWebCompaction: true,
+  // Opt-in since v0.6: never rewrite presets unless the user asks.
+  autoWebCompaction: false,
   // Memory-to-memory semantics (zero LLM): when ON, an entity-anchored write
   // that carries a replacement/negation cue ("改用 / no longer / switched…")
   // auto-marks the matching older same-entity+kind memory as superseded.
@@ -145,9 +165,13 @@ const DEFAULTS = {
   // (deterministic surface-replace, zero LLM). L0 simulation (77 sessions):
   // todo-progress boundaries cover 79% of replay exposure; the repayment
   // gate below lifted cache-aware net savings 12–32% on every threshold.
-  // Default OFF (SoL-Pi C13: a dormant mechanism costs nothing); the
-  // pruner service is feature-detected — absent ⇒ disabled with one log.
-  boundaryPrune: false,
+  // Default ON since v0.6 (PROPOSAL-evolution §3.5): `false` is the opt-out;
+  // the pruner service is feature-detected — absent ⇒ disabled with one log.
+  boundaryPrune: true,
+  // Adaptive off-switch (v0.6-B): the Context Ledger vetoes further fires in a
+  // workspace whose own accounting is net negative after real fires. Evidence
+  // can only turn pruning OFF — never on. `false` = repayment gate only.
+  boundaryPruneAdaptive: true,
   // Repayment gate: prune only while the session is expected to run at
   // least this many more requests (empirical workspace prior, see
   // lib/boundary-prune.js). 50 = L0 sweep optimum.
@@ -170,9 +194,10 @@ const DEFAULTS = {
   // append a `[middle archived: result_recall(<callId>) 可取]` handle tail to
   // each pruned node; register `result_recall(callId, offset?)` for paged
   // recall — the full ObservationPack pattern (SoL-Pi mech ③; paired
-  // EdgeBench −23.58%). Requires `boundaryPrune: true` to have anything to
-  // archive (L2 without L1 is a no-op with a log line). Default OFF (C13).
-  resultPack: false,
+  // EdgeBench −23.58%). Requires boundary pruning to have anything to
+  // archive (L2 without L1 is a no-op with a log line). Default ON since v0.6:
+  // default pruning must be lossless (the archive is the recovery path).
+  resultPack: true,
   // result_recall page budget (chars per page). 4000 fits a typical model
   // context band without re-materializing huge results whole.
   resultPackPageChars: 4000,
@@ -304,6 +329,55 @@ export function apply(ctx, config) {
   // todo_write promote hook. Process-local; fed by a tools/result listener.
   const recorder = makeTriggerRecorder(resolved);
 
+  // Context Ledger: created before the service object so /stats can read it.
+  // `ledgerEnabled:false` leaves a null instance (zero listeners, zero writes).
+  const ledger = resolved.ledgerEnabled === false
+    ? null
+    : createLedger({
+      home: engramHome(),
+      cacheDiscount: resolved.ledgerCacheDiscount,
+      log,
+    });
+
+  // Ledger-driven off-switch for BoundaryPrune (PROPOSAL-evolution v0.6-B):
+  // per-workspace, 30s-cached, reads the same summary /stats exposes. Evidence
+  // can only turn pruning OFF. No ledger → allow (repayment gate still runs).
+  const boundaryGate = (() => {
+    const cache = new Map();
+    const TTL_MS = 30_000;
+    return (workspace) => {
+      if (ledger === null) return true;
+      const now = Date.now();
+      const hit = cache.get(workspace);
+      if (hit !== void 0 && now - hit.at < TTL_MS) return hit.ok;
+      let ok = true;
+      try {
+        const summary = ledger.summary();
+        const bucket = summary.byWorkspace[workspace] ?? summary.totals;
+        ok = !(bucket.rewriteCount >= 3 && bucket.netChars < 0);
+      } catch {
+        ok = true; // accounting must never disable the mechanism by accident
+      }
+      if (cache.size > 64) cache.clear();
+      cache.set(workspace, { at: now, ok });
+      return ok;
+    };
+  })();
+
+  // JIT push (v0.6-C): append-only hints, priced through the ledger.
+  const jit = resolved.jitEnabled === false
+    ? null
+    : createJit({
+      maxLines: resolved.jitMaxLines,
+      maxLineChars: resolved.jitMaxLineChars,
+      log,
+      onPush: ({ session, workspace, chars }) => {
+        ledger?.noteRewrite(session?.id, workspace, {
+          chars: estimatePrefixChars(session) + chars,
+        });
+      },
+    });
+
   const service = {
     config: resolved,
     getDomain,
@@ -319,6 +393,10 @@ export function apply(ctx, config) {
     /** SoL-Pi activation telemetry (M15/C13): durable boundary-prune fires +
      *  [RECALL] injection funnel, read at request time (readPackTelemetry). */
     packTelemetry: () => readPackTelemetry({ home: engramHome() }),
+    /** Context Ledger snapshot: spent vs saved context, net of cache rewrites. */
+    ledger: () => (ledger === null ? null : ledger.summary()),
+    /** JIT push counters (per process; /stats diagnostic surface). */
+    jitStats: () => (jit === null ? null : jit.stats()),
     /** Last GC sweep totals (GUI overview observability). */
     gcStats: { lastRun: 0, archivedMemories: 0, archivedTasks: 0, removedLinks: 0 },
     /** Live Context GC status (API overview + durable status file). */
@@ -452,6 +530,7 @@ export function apply(ctx, config) {
     // [RECALL] block.
     if (pendingRecall) return block;
     sessionCache.set(session, block);
+    ledger?.noteInjection(session.id, wk(cwd), "index", block.length);
     return block;
   };
 
@@ -500,9 +579,22 @@ export function apply(ctx, config) {
     } catch (error) {
       log?.warn?.(`engram esr render failed: ${String(error)}`);
     }
+    if (block.length > 0) ledger?.noteInjection(session.id, wk(cwd), "esr", block.length);
     return block;
   };
 
+  // JIT block: append-only lines accumulated by lib/jit.js (never edited).
+  const jitFor = (agent) => {
+    const session = agent?.session;
+    if (session === void 0) return "";
+    const text = jit === null ? "" : jit.textFor(session);
+    const cwd = session.header?.cwd;
+    if (text.length > 0 && typeof cwd === "string" && cwd.length > 0) {
+      ledger?.noteInjection(session.id, wk(cwd), "jit", text.length);
+    }
+    return text;
+  };
+
   ctx.effect(() => {
     const disposers = [];
 
@@ -649,6 +741,18 @@ export function apply(ctx, config) {
       });
       disposers.push(() => indexDispose());
 
+      // JIT push block — empty until a deterministic event fires, then
+      // append-only (prefix changes at most once per push).
+      const jitDispose = systemPrompt.section({
+        name: "engram:jit",
+        order: resolved.engramJitOrder,
+        text: (assemblyContext) => {
+          const agent = assemblyContext?.agent;
+          return agent === void 0 ? "" : jitFor(agent);
+        },
+      });
+      disposers.push(() => jitDispose());
+
       // Static ESR protocol (pi-esr rule 1): byte-identical every turn, so
       // the model never depends on runtime nudges to know WHEN to use ESR.
       const esrMethodDispose = systemPrompt.section({
@@ -677,6 +781,39 @@ export function apply(ctx, config) {
     });
     disposers.push(disposeTrigger);
 
+    // Context Ledger ticks: one model request completed → charge the injected
+    // blocks (held in the prefix) and credit pruned/compacted chars that would
+    // otherwise replay in this request. Session teardown folds the tail into
+    // the daily rollup. Both listeners are no-ops when the ledger is disabled.
+    if (ledger !== null) {
+      const disposeLedgerTick = ctx.on("session/event", (session, event) => {
+        if (event?.type !== "assistant/message") return;
+        ledger.tick(session?.id, wk(session?.header?.cwd));
+      });
+      disposers.push(disposeLedgerTick);
+      const disposeLedgerSession = ctx.on("session/disposed", (session) => {
+        ledger.dispose(session?.id);
+      });
+      disposers.push(disposeLedgerSession);
+    }
+
+    // JIT observers: tool results (error recurrence / file focus) and goal
+    // completion. `opened` may be undefined early — the observers no-op.
+    if (jit !== null) {
+      const disposeJitTool = ctx.on("tools/result", (exec, result) => {
+        jit.observeToolResult(exec, result, opened);
+      });
+      disposers.push(disposeJitTool);
+      const disposeJitEvent = ctx.on("session/event", (session, event) => {
+        jit.observeEvent(session, event, opened);
+      });
+      disposers.push(disposeJitEvent);
+      const disposeJitSession = ctx.on("session/disposed", (session) => {
+        jit.dispose(session?.id);
+      });
+      disposers.push(disposeJitSession);
+    }
+
     // session-end todo auto-sink: a session whose plan still has pending todos
     // lands them as ESR drafts at its teardown edge, so the plan survives the
     // session (flip `autoSinkTodosOnEnd` off to opt out). Live-toggleable — the
@@ -765,14 +902,23 @@ export function apply(ctx, config) {
         // independent of `resultPack`: /stats must be able to prove the L1
         // mechanism actually fires on the live profile.
         onPruned: (session, report, boundary) => {
+          const workspace = wk(session?.header?.cwd);
           recordBoundaryPruneFire({
             home: engramHome(),
             boundary,
             report,
-            workspace: wk(session?.header?.cwd),
+            workspace,
+          });
+          // Ledger: the removed chars stop replaying from the next request;
+          // the prefix invalidation is priced with the pre-prune size (the
+          // surface is already rewritten when this hook runs).
+          const removed = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
+          const prefixChars = estimatePrefixChars(session);
+          ledger?.notePrune(session?.id, workspace, {
+            charsRemoved: removed,
+            rewriteChars: prefixChars + removed,
           });
           if (resolved.resultPack !== true) return;
-          const workspace = wk(session?.header?.cwd);
           archivePrunedResults({
             dir: l2Dir,
             session,
@@ -781,15 +927,16 @@ export function apply(ctx, config) {
             log,
           });
         },
-      });
+      }, { allowFire: boundaryGate });
       disposers.push(disposeBoundaryPrune);
 
-      // ResultPack L2 tool (default OFF with L1): `result_recall(callId, offset?)`
-      // pages back an archived middle. Independent of the memory domain — the
-      // archive lives on the filesystem. Registers only when enabled; without
-      // boundayPrune there is nothing to archive, so it self-disables.
+      // ResultPack L2 tool (default ON with L1 since v0.6): `result_recall(callId,
+      // offset?)` pages back an archived middle. Independent of the memory
+      // domain — the archive lives on the filesystem. Registers only when
+      // enabled; without boundaryPrune there is nothing to archive, so it
+      // self-disables.
       if (resolved.resultPack === true) {
-        if (resolved.boundaryPrune !== true) {
+        if (resolved.boundaryPrune === false) {
           log?.warn?.("engram result-pack: requires boundaryPrune: true to prune middles — result_recall not registered");
         } else {
           const pageChars = Number.isFinite(resolved.resultPackPageChars)

package/lib/jit.js

@@ -0,0 +1,224 @@
+/**
+ * dsh-engram JIT push (PROPOSAL-evolution §4 v0.6-C).
+ *
+ * The measured failure mode: pull-only recall is dormant — the model rarely
+ * calls `engram_recall` on its own (24 calls / 8 details across 112 real
+ * sessions). JIT flips the direction: when a deterministic, high-precision
+ * event makes a known memory immediately relevant, append ONE short line to
+ * the prompt at that moment:
+ *
+ *   - error recurrence — the same `cmd:` signature fails a second time, and a
+ *     prior error/procedure memory already exists for it;
+ *   - file focus — the same file is edited twice in a row, and a memory is
+ *     anchored to that file (preference > fact/decision);
+ *   - goal completion — a goal completes/blocks while active ESR tasks remain.
+ *
+ * Invariants:
+ *   - append-only per session: a pushed line is never edited or withdrawn, so
+ *     the prefix changes at most once per push (the ledger prices the rewrite);
+ *   - no hit, no line (silence beats noise);
+ *   - hard budget: `maxLines` per session, `maxLineChars` per line;
+ *   - pure rules, zero LLM, fail-contained.
+ *
+ * The resume brief (§v0.7) is deliberately not here yet.
+ */
+
+import { cmdTag } from "./capture.js";
+import { shortId, workspaceKey as wk } from "./util.js";
+
+const EDIT_TOOLS = new Set(["write", "str_replace_editor", "edit"]);
+const RUN_TOOLS = new Set(["bash", "shell", "exec"]);
+
+function clip(text, max) {
+  const flat = String(text ?? "").replace(/\s+/g, " ").trim();
+  return flat.length <= max ? flat : `${flat.slice(0, max - 1)}…`;
+}
+
+function basename(p) {
+  const parts = String(p ?? "").split("/");
+  return parts[parts.length - 1] || String(p ?? "");
+}
+
+function dirname(p) {
+  const parts = String(p ?? "").split("/");
+  parts.pop();
+  return parts.join("/");
+}
+
+function isMemory(row) {
+  return row !== null && typeof row === "object" && typeof row.id === "string";
+}
+
+/**
+ * @param {object} [opts]
+ * @param {number} [opts.maxLines] hard cap per session (default 3)
+ * @param {number} [opts.maxLineChars] hard cap per line (default 220)
+ * @param {(push: {sessionId: string, workspace: string, session: object, line: string, chars: number, kind: string}) => void} [opts.onPush]
+ * @param {object} [opts.log]
+ */
+export function createJit({ maxLines = 3, maxLineChars = 220, onPush = null, log = void 0 } = {}) {
+  const lineCap = Number.isFinite(maxLines) && maxLines >= 0 ? Math.trunc(maxLines) : 3;
+  const charCap = Number.isFinite(maxLineChars) && maxLineChars > 0 ? Math.trunc(maxLineChars) : 220;
+  const states = new Map();
+  const totals = { pushes: 0, byKind: { error: 0, focus: 0, goal: 0 } };
+
+  function stateFor(sessionId, workspace) {
+    let state = states.get(sessionId);
+    if (state === void 0) {
+      state = {
+        sessionId,
+        workspace: typeof workspace === "string" ? workspace : "",
+        lines: [],
+        seen: new Set(),
+        failures: new Map(),
+        lastFile: "",
+        sameFile: 0,
+        focusSent: new Set(),
+        goalSent: false,
+      };
+      states.set(sessionId, state);
+    } else if (typeof workspace === "string" && workspace.length > 0) {
+      state.workspace = workspace;
+    }
+    return state;
+  }
+
+  function push(state, session, kind, line) {
+    if (lineCap === 0 || state.lines.length >= lineCap) return false;
+    const text = clip(line, charCap);
+    if (text.length === 0 || state.seen.has(text)) return false;
+    state.lines.push(text);
+    state.seen.add(text);
+    totals.pushes += 1;
+    totals.byKind[kind] = (totals.byKind[kind] ?? 0) + 1;
+    try {
+      onPush?.({
+        sessionId: state.sessionId,
+        workspace: state.workspace,
+        session,
+        kind,
+        line: text,
+        chars: text.length,
+      });
+    } catch (error) {
+      log?.warn?.(`engram jit onPush failed: ${String(error)}`);
+    }
+    return true;
+  }
+
+  function failLine(state, session, domain, tag) {
+    const rows = domain?.searchMemories?.({ workspace: state.workspace, tag, status: "active", limit: 8 }) ?? [];
+    const memory = rows.find((row) => isMemory(row) && row.kind === "procedure" && /^fixed:/.test(row.text ?? ""))
+      ?? rows.find((row) => isMemory(row) && row.kind === "error");
+    if (memory === void 0) return false;
+    return push(
+      state,
+      session,
+      "error",
+      `[JIT] ${clip(tag, 40)} failed again ×2 — ${memory.kind} #${shortId(memory.id)}: ${clip(memory.text, 92)} · engram_detail ${memory.id}`,
+    );
+  }
+
+  function focusLine(state, session, domain, path) {
+    if (state.focusSent.has(path)) return false;
+    state.focusSent.add(path);
+    const rows = domain?.listMemories?.(state.workspace, 300) ?? [];
+    const dirTag = `dir:${dirname(path)}`;
+    const anchored = rows.filter(
+      (row) => isMemory(row)
+        && row.status === "active"
+        && (row.filePath === path || (Array.isArray(row.tags) && row.tags.includes(dirTag))),
+    );
+    const memory = anchored.find((row) => row.kind === "preference")
+      ?? anchored.find((row) => ["fact", "decision", "procedure"].includes(row.kind));
+    if (memory === void 0) return false;
+    return push(
+      state,
+      session,
+      "focus",
+      `[JIT] focus ${clip(basename(path), 36)} ×2 — ${memory.kind} #${shortId(memory.id)}: ${clip(memory.text, 92)} · engram_detail ${memory.id}`,
+    );
+  }
+
+  return {
+    /** Observe a tool result; may push an error-recurrence or file-focus line. */
+    observeToolResult(exec, result, domain) {
+      try {
+        const session = exec?.agent?.session;
+        const sessionId = session?.id;
+        const cwd = session?.header?.cwd;
+        if (typeof sessionId !== "string" || sessionId.length === 0) return;
+        if (typeof cwd !== "string" || cwd.length === 0) return;
+        const state = stateFor(sessionId, wk(cwd));
+        const name = String(exec?.name ?? "").toLowerCase();
+        const args = exec?.arguments ?? {};
+
+        if (RUN_TOOLS.has(name) && result?.isError === true && typeof args.command === "string") {
+          const tag = cmdTag(args.command);
+          if (tag === null) return;
+          const count = (state.failures.get(tag) ?? 0) + 1;
+          state.failures.set(tag, count);
+          if (count === 2) failLine(state, session, domain, tag);
+          return;
+        }
+
+        if (EDIT_TOOLS.has(name) && result?.isError !== true) {
+          const path = String(args.path ?? args.file_path ?? args.filePath ?? "");
+          if (path.length === 0) return;
+          if (state.lastFile === path) state.sameFile += 1;
+          else {
+            state.lastFile = path;
+            state.sameFile = 1;
+          }
+          if (state.sameFile === 2) focusLine(state, session, domain, path);
+        }
+      } catch (error) {
+        log?.warn?.(`engram jit tool observe threw: ${String(error)}`);
+      }
+    },
+
+    /** Observe a session event; may push a goal-completion line. */
+    observeEvent(session, event, domain) {
+      try {
+        if (event?.type !== "goal/change") return;
+        const data = event?.data;
+        if (data === null || typeof data !== "object") return;
+        if (data.operation !== "complete" && data.operation !== "block") return;
+        const goal = data.goal;
+        if (goal === null || typeof goal !== "object") return;
+        const sessionId = session?.id;
+        const cwd = session?.header?.cwd;
+        if (typeof sessionId !== "string" || sessionId.length === 0) return;
+        if (typeof cwd !== "string" || cwd.length === 0) return;
+        const state = stateFor(sessionId, wk(cwd));
+        if (state.goalSent) return;
+        const tasks = domain?.listTasks?.(state.workspace) ?? [];
+        const active = tasks.filter((task) => task?.state === "active");
+        if (active.length === 0) return;
+        const shown = active.slice(0, 2).map((task) => `${task.id} ${clip(task.name, 48)}`).join("; ");
+        const more = active.length > 2 ? ` +${active.length - 2}` : "";
+        state.goalSent = true;
+        push(state, session, "goal", `[JIT] goal ${data.operation} — open: ${shown}${more} · esr_status`);
+      } catch (error) {
+        log?.warn?.(`engram jit event observe threw: ${String(error)}`);
+      }
+    },
+
+    /** The frozen JIT block for this session ("" when nothing was pushed). */
+    textFor(session) {
+      const sessionId = session?.id;
+      if (typeof sessionId !== "string") return "";
+      return states.get(sessionId)?.lines.join("\n") ?? "";
+    },
+
+    /** Drop per-session state at teardown. */
+    dispose(sessionId) {
+      states.delete(sessionId);
+    },
+
+    /** Push counters for /stats + diagnostics. */
+    stats() {
+      return { ...totals, byKind: { ...totals.byKind }, sessions: states.size };
+    },
+  };
+}