dsh-engram 0.3.4 → 0.3.6

package/lib/context-gc.js

@@ -12,12 +12,18 @@
  * with no re-fetchable backing) get a scoped LLM narrative — the safety net,
  * configurable off for a pure-mechanical zero-LLM path.
  *
- * The 6 pi-esr GC constraints:
+ * The 6 pi-esr GC constraints (+ the harness shrink gate that keeps them legal):
  *   - indexed-output-evictable → pointer to `engram_detail(id)` / `engram_recall`
  *   - stable-task-evictable    → pointer to the [ESR] block / `esr_ready`
  *   - working-set-protected    → active tasks restated in the summary
  *   - pointer-salience         → every evicted category has an explicit re-fetch call
  *   - no-provenance-no-evict   → un-provenanced turns via scoped narrative or verbatim
+ *   - shrink-gated             → the framed checkpoint must be STRICTLY SMALLER
+ *     than the evicted span (`fitSummaryToSpan`): the harness refuses and rolls
+ *     back any summary that is not smaller, which otherwise blocks every auto
+ *     compaction and pushes the session into the model-side context overflow.
+ *     The pointer/narrative body is therefore fitted under the span with the
+ *     host token meter, truncating tail detail (still durable in the log).
  *
  * Layout: the pure (zero-LLM, dependency-free) layer lives here; the DSH
  * `CompactionEngine` binding is `loadContextGcEngine`, behind a guarded dynamic
@@ -67,6 +73,152 @@ export function blockText(content) {
   return out.trim();
 }
 
+/**
+ * The durable checkpoint framing the harness applies to a compaction summary
+ * before accepting it (`frameSummary()` in dsh-compaction-basic): the summary
+ * body is wrapped in a preamble + `<compacted-summary>` tags and the WHOLE
+ * framed user message must estimate strictly SMALLER than the evicted span,
+ * or the harness throws `summary is not smaller than the shadowed content`
+ * and rolls the compaction back. These constants mirror that framing so the
+ * Context GC can self-predict acceptance with the host token meter (verified
+ * to reproduce the harness's estimate exactly); a small safety margin absorbs
+ * any harness drift.
+ */
+const FRAME_PREAMBLE =
+  "This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.";
+const FRAME_OPEN_TAG = "<compacted-summary>";
+const FRAME_CLOSE_TAG = "</compacted-summary>";
+
+/** Best-effort token estimate of one derived message through the host meter. */
+function estimateMessageTokens(meter, message) {
+  if (typeof meter?.estimateMessage !== "function") return 0;
+  try {
+    const tokens = meter.estimateMessage(message);
+    return Number.isFinite(tokens) && tokens > 0 ? tokens : 0;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Exact token estimate of the span a compaction is about to shadow: the sum of
+ * the evicted region's derived messages through the SAME token meter the
+ * harness compares against (verified to equal the harness's shadowed count).
+ * Returns 0 when no meter is available — callers then skip the shrink gate
+ * rather than guess.
+ * @param {object|undefined} meter - `ctx.tokenMeter` (host), or undefined.
+ * @param {Array<object>} messages - the region messages passed to `summarize`.
+ * @returns {number}
+ */
+export function estimateShadowedTokens(meter, messages) {
+  const list = Array.isArray(messages) ? messages : [];
+  let total = 0;
+  for (const message of list) total += estimateMessageTokens(meter, message);
+  return total;
+}
+
+/** The harness-framed checkpoint user message for a summary body. */
+function framedCheckpointMessage(bodyText) {
+  return {
+    role: "user",
+    content: [
+      { type: "text", text: `${FRAME_PREAMBLE}\n\n${FRAME_OPEN_TAG}` },
+      { type: "text", text: bodyText },
+      { type: "text", text: FRAME_CLOSE_TAG },
+    ],
+  };
+}
+
+/**
+ * Token estimate of the framed checkpoint the harness will generate for this
+ * body (mirrors `frameSummary` + `createUserMessage` + `estimateMessage`;
+ * verified byte-identical to the live meter). Without a meter, a conservative
+ * chars→tokens heuristic is used — divided by two so both CJK and latin
+ * over-estimate, biasing toward trimming more, never less.
+ * @param {object|undefined} meter - `ctx.tokenMeter`, or undefined.
+ * @param {string} bodyText - the summary body inside the checkpoint.
+ * @returns {number}
+ */
+export function estimateFramedTokens(meter, bodyText) {
+  if (typeof meter?.estimateMessage === "function") {
+    try {
+      const tokens = meter.estimateMessage(framedCheckpointMessage(bodyText));
+      if (Number.isFinite(tokens)) return Math.max(1, tokens);
+    } catch {
+      // fall through to the char heuristic
+    }
+  }
+  const text = typeof bodyText === "string" ? bodyText : "";
+  return (
+    Math.ceil((FRAME_PREAMBLE.length + FRAME_OPEN_TAG.length + FRAME_CLOSE_TAG.length + text.length) / 2) + 8
+  );
+}
+
+/** Truncation note appended when the checkpoint must shrink to fit the span. */
+const FIT_SUFFIX = "\n_…(full detail stays re-fetchable via the engram/ESR tools; see the session log)_";
+
+/**
+ * Shrink a Context GC summary body so the framed checkpoint stays strictly
+ * under the evicted span — the harness's hard shrink gate. Without this, a
+ * large verbatim/pointer body makes every automatic pressure / overflow
+ * compaction roll back (`summary is not smaller than the shadowed content`),
+ * the context never shrinks, and the session eventually hits the model-side
+ * context overflow.
+ *
+ * Truncation keeps the largest PREFIX that still fits — the pointer head and
+ * working set survive, only tail detail is cut — and appends a short note when
+ * the span is large enough to afford one. Exact under the host token meter;
+ * deterministic everywhere; degrades to an empty body for spans so small that
+ * the exact framing overhead does not fit (the stock LLM summarizer would be
+ * refused by the harness for those too).
+ *
+ * @param {string} bodyText - the full Context GC body.
+ * @param {number} shadowedTokens - evicted-span token estimate (0 = unknown).
+ * @param {{meter?: object, margin?: number}} [opts]
+ * @returns {{ text: string, framed: number, shadowed: number, trimmed: boolean }}
+ */
+export function fitSummaryToSpan(bodyText, shadowedTokens, opts = {}) {
+  const meter = opts?.meter;
+  const margin = opts?.margin ?? 4;
+  const budget = shadowedTokens - margin;
+  if (!Number.isFinite(shadowedTokens) || shadowedTokens <= 0 || budget <= 1) {
+    return {
+      text: bodyText,
+      framed: estimateFramedTokens(meter, bodyText),
+      shadowed: shadowedTokens,
+      trimmed: false,
+    };
+  }
+  const source = typeof bodyText === "string" && bodyText.length > 0 ? bodyText : "";
+  if (estimateFramedTokens(meter, source) < budget) {
+    return {
+      text: source,
+      framed: estimateFramedTokens(meter, source),
+      shadowed: shadowedTokens,
+      trimmed: false,
+    };
+  }
+  // Largest prefix of the source that fits WITHOUT the note; the note is only
+  // re-added when the budget still affords it (the note itself is ~25 tokens).
+  let lo = 0;
+  let hi = source.length;
+  let best = 0;
+  while (lo <= hi) {
+    const mid = Math.floor((lo + hi) / 2);
+    if (estimateFramedTokens(meter, source.slice(0, mid)) < budget) {
+      best = mid;
+      lo = mid + 1;
+    } else {
+      hi = mid - 1;
+    }
+  }
+  const prefix = source.slice(0, best);
+  const withNote = `${prefix}${FIT_SUFFIX}`;
+  const text = best > 0 && estimateFramedTokens(meter, withNote) < budget ? withNote : prefix;
+  const trimmed = text !== source;
+  return { text, framed: estimateFramedTokens(meter, text), shadowed: shadowedTokens, trimmed };
+}
+
 function addSet(set, value) {
   const v = String(value ?? "").trim();
   if (v.length > 0) set.add(v);
@@ -466,6 +618,57 @@ export async function loadCompactionEngine(ctx, config = {}, deps = {}, { mode =
   return loadContextGcEngine(ctx, config, deps);
 }
 
+/**
+ * Mount the compaction engine onto `ctx` — THE shared entry both planes use
+ * (host plane: the main engram plugin's `apply`; preset plane: the
+ * `dsh-engram/compaction` row shown in an agent preset's `compaction` group).
+ *
+ * `config.gcReplacesCompaction === false` → mount the bare default LLM
+ * summarizer instead (keeps the `compaction` service alive after the profile
+ * patch disabled the base `compaction-basic` row); otherwise mount the
+ * ContextGcEngine override. `deps.readWorkspace(ws, report)` is how the engine
+ * reaches engram/ESR state for pointer classification — the caller supplies
+ * its own lazy-domain binding.
+ *
+ * Fully contained: an unimportable backend or a duplicate `compaction`
+ * provider (cordis refuses second registration) only warns and returns null —
+ * the host keeps whatever compaction it already has, never a crash.
+ *
+ * @returns {Promise<Function|null>} the mounted engine class, or null.
+ */
+export async function mountCompactionEngine(ctx, config = {}, deps = {}) {
+  const mode = config.gcReplacesCompaction === false ? "default" : "context-gc";
+  const engineDeps =
+    mode === "context-gc"
+      ? {
+          narrativeEnabled: config.gcNarrative !== false,
+          narrativeMaxTokens: config.gcNarrativeMaxTokens ?? 1024,
+          narrativeMaxChars: config.gcNarrativeMaxChars ?? 4000,
+          ...deps,
+        }
+      : {};
+  const Engine = await loadCompactionEngine(ctx, { auto: true }, engineDeps, { mode });
+  if (!Engine) {
+    ctx?.logger?.warn?.("engram context-gc: dsh-compaction-basic unavailable — no compaction engine");
+    return null;
+  }
+  try {
+    new Engine();
+  } catch (error) {
+    // Duplicate `compaction` provider (base row not disabled) or constructor
+    // refusal: keep whatever is already registered, never crash the host.
+    ctx?.logger?.warn?.(
+      `engram context-gc mount skipped (${error instanceof Error ? error.message : String(error)}) — keeping the existing compaction service`,
+    );
+    return null;
+  }
+  ctx?.logger?.info?.(
+    `engram context-gc: compaction = ${mode === "context-gc" ? "Context GC (mechanical eviction + re-fetch pointers)" : "default LLM summarizer (gcReplacesCompaction:false)"}`,
+  );
+  return Engine;
+}
+
+
 /**
  * Load the DSH binding: a `ContextGcEngine extends BasicCompactionEngine`
  * whose sole override is `summarize()`. Returns `null` when
@@ -542,9 +745,38 @@ export async function loadContextGcEngine(ctx, config = {}, deps = {}) {
         callLlm,
       });
 
-      const summaryText = buildContextGcSummary(report, safeContext, { turns, narrative });
+      let summaryText = buildContextGcSummary(report, safeContext, { turns, narrative });
+
+      // The harness refuses any checkpoint whose framed token estimate is >=
+      // the evicted span (see fitSummaryToSpan). Fit the body under that span
+      // so automatic pressure / context-overflow compaction COMMITS instead of
+      // rolling back — otherwise the session only grows until the model-side
+      // context overflow fires.
+      const meter = this.ctx?.tokenMeter;
+      const shadowedTokens = estimateShadowedTokens(meter, messages);
+      const fit = fitSummaryToSpan(summaryText, shadowedTokens, { meter });
+      if (fit.trimmed && typeof this.ctx?.logger?.info === "function") {
+        this.ctx.logger.info(
+          `engram context-gc: checkpoint trimmed to fit the evicted span (${fit.framed} framed vs ${fit.shadowed} shadowed tokens)`,
+        );
+      }
+      summaryText = fit.text;
+
       const result = { summary: [{ type: "text", text: summaryText }] };
       if (narrative.llm && typeof narrative.llm === "object") Object.assign(result, narrative.llm);
+      // The harness's durable `compaction/summary` record ALWAYS carries
+      // `provider`/`model` (spread unconditionally; undefined values reject the
+      // whole event as non-JSON-serializable). The mechanical path makes no LLM
+      // call, so stamp the conversation's routed target — the same model the
+      // default summarizer would have used — as the record's provenance.
+      const target = this._narrativeTarget(agent);
+      if (target && typeof target.provider === "string" && typeof target.model === "string") {
+        result.provider = result.provider ?? target.provider;
+        result.model = result.model ?? target.model;
+      }
+      if (result.maxTokens === undefined && this.config && Number.isFinite(this.config.maxTokens)) {
+        result.maxTokens = this.config.maxTokens;
+      }
       return result;
     }
 

package/lib/goal-capture.js

@@ -0,0 +1,89 @@
+/**
+ * dsh-engram: DSH goal-domain integration.
+ *
+ * The harness's goal service persists a durable `goal/change` session event on
+ * every goal mutation. This module listens for those events and turns the two
+ * terminal states into memories, so a completed / blocked goal survives the
+ * session — deterministic, zero LLM, purely mechanical:
+ *
+ *   - complete → `handoff` memory (high signal, tag `goal`)
+ *   - block    → `error` memory (tag `goal` + `blocked`) — the blocker is a
+ *                real, recall-worthy signal
+ *
+ * The goal snapshot fields are read verbatim; nothing here calls a model and
+ * the listener never throws into the session pipeline (fully contained).
+ */
+
+import { workspaceKey as wk } from "./util.js";
+
+function isRecord(v) {
+  return v !== null && typeof v === "object" && !Array.isArray(v);
+}
+
+/**
+ * Pure reducer: map a `goal/change` session-event payload to a memory entry,
+ * or null when the event is not a terminal goal state worth sedimenting.
+ * Deterministic and unit-testable without the harness.
+ */
+export function goalChangeToMemory(change) {
+  if (!isRecord(change) || change.kind !== "goal/change") return null;
+  if (change.operation !== "complete" && change.operation !== "block") return null;
+  const goal = change.goal;
+  if (!isRecord(goal) || typeof goal.objective !== "string") return null;
+  const objective = goal.objective.trim().slice(0, 400);
+  if (objective.length === 0) return null;
+  if (change.operation === "complete") {
+    return {
+      kind: "handoff",
+      text: `goal complete: ${objective}`,
+      tags: ["goal", "auto-captured", "complete"],
+      signal: 0.6,
+    };
+  }
+  const reason = typeof goal.blockedReason === "string" && goal.blockedReason.trim().length > 0
+    ? ` — ${goal.blockedReason.trim().slice(0, 160)}`
+    : "";
+  return {
+    kind: "error",
+    text: `goal blocked: ${objective}${reason}`,
+    tags: ["goal", "auto-captured", "blocked"],
+    signal: 0.4,
+  };
+}
+
+/**
+ * Install the `session/event` listener that auto-sediments terminal goals.
+ * `storeAdapter` is any { storeMemory(entry) -> Promise } face (the same lazy
+ * adapter auto-capture uses); it is resolved per event, so early events that
+ * beat the first domain open are simply dropped. Returns the disposer.
+ */
+export function installGoalCapture(ctx, storeAdapter, log) {
+  const onEvent = (session, event) => {
+    try {
+      if (event?.type !== "goal/change") return;
+      const entry = goalChangeToMemory(event.data);
+      if (entry === null) return;
+      const cwd = session?.header?.cwd;
+      const sessionId = session?.id;
+      if (typeof cwd !== "string" || cwd.length === 0) return;
+      if (typeof sessionId !== "string" || sessionId.length === 0) return;
+      storeAdapter
+        .storeMemory({
+          workspace: wk(cwd),
+          kind: entry.kind,
+          text: entry.text,
+          tags: entry.tags,
+          signal: entry.signal,
+          filePath: null,
+          sessionId,
+          seq: 0,
+        })
+        .then(() => {})
+        .catch((error) => log?.warn?.(`engram goal capture failed: ${String(error)}`));
+    } catch (error) {
+      // fully contained — a capture bug must never disturb the session pipeline
+      log?.warn?.(`engram goal capture threw: ${String(error)}`);
+    }
+  };
+  return ctx.on("session/event", onEvent);
+}

package/lib/index-block.js

@@ -51,7 +51,11 @@ export function isProvenProcedure(m, config) {
  */
 export function selectIndexMemories(store, workspace, config) {
   const all = store.listMemories(workspace, 200);
+  // Stale truth (a newer memory marked itself as replacing that row) never
+  // earns an index line — the current statement is the one worth surfacing.
+  const { supersededBy } = store.memoryRelations?.(workspace) ?? { supersededBy: new Map() };
   return all
+    .filter((m) => !supersededBy.has(m.id))
     .filter(
       (m) =>
         (isGitPlumbingEcho(m) && (m.hits ?? 0) < config.promoteHits
@@ -111,7 +115,7 @@ export function renderIndex(store, workspace, cwd, config) {
   if (entityNames.length > 0) lines.push(`nodes: ${truncate(entityNames.join(", "), 72)}`);
   for (const m of memories.slice(0, config.indexMaxLines)) lines.push(memoryLine(m, config));
   for (const t of tasks.slice(0, 4)) lines.push(taskStateLine(t));
-  lines.push("drill: engram_recall <query> | engram_detail <id> | esr_task (multi-step work → create now) | esr_node (recurring domain object → create now) | esr_link (relate nodes/tasks)");
+  lines.push("drill: use [RECALL] below · engram_detail <id> (full record) · engram_recall <query> (more) · esr_task/esr_node/esr_link (work)");
   return enforceCharCap(lines, config.indexMaxChars);
 }
 
@@ -129,6 +133,97 @@ function enforceCharCap(lines, maxChars) {
   return joined;
 }
 
+/** Text of the session's first user message (the opening instruction of the
+ *  conversation). Walk `session.events` for the first `user/message` event and
+ *  join its `data.content` text blocks; fall back to the first text-bearing
+ *  event ("goal"/reminder start) guardedly. Returns "" when there is none.
+ *  Pure, deterministic — the auto-recall (B) keys its query off this. */
+export function firstUserMessageText(session) {
+  if (session === null || typeof session !== "object") return "";
+  const events = Array.isArray(session.events) ? session.events : [];
+  if (events.length === 0) return "";
+  const textOf = (data) => {
+    if (data === null || typeof data !== "object") return "";
+    const blocks = Array.isArray(data.content) ? data.content : [];
+    if (blocks.length > 0) {
+      return blocks
+        .map((b) => (b !== null && typeof b === "object" && typeof b.text === "string" ? b.text : ""))
+        .join(" ")
+        .trim();
+    }
+    return typeof data.text === "string" ? data.text.trim() : "";
+  };
+  for (const event of events) {
+    if (event === null || typeof event !== "object") continue;
+    if (event.type !== "user/message") continue;
+    const text = textOf(event.data);
+    if (text.length > 0) return text;
+  }
+  // Fallback: no user/message yet — first event carrying text (goal-driven,
+  // headless or reminder-seeded starts) still gives the recall a key.
+  for (const event of events) {
+    if (event === null || typeof event !== "object" || event.type === "user/message") continue;
+    const text = textOf(event.data);
+    if (text.length > 0) return text;
+  }
+  return "";
+}
+
+/**
+ * Host-side session-start auto-recall (方案 B): at the first assembly of a new
+ * session, deterministically key a bounded BM25 recall off the session's first
+ * user message and inject the top hits as a `[RECALL]` block — so relevant
+ * memory reaches the model even when it never calls `engram_recall` on its own
+ * (the observed real-world gap). Pure rule, zero LLM, no `hits` bumps (a pure
+ * read). Falls back to the newest high-signal memories when the query is empty
+ * and `allowRecencyFallback` is on. Returns "" when nothing useful to show.
+ */
+export function renderStartRecall(store, workspace, config = {}, opts = {}) {
+  if (store === null || typeof store?.recall !== "function") return "";
+  const limit = Math.max(1, Math.min(8, Number.isFinite(config.autoRecallLimit) ? Math.trunc(config.autoRecallLimit) : 3));
+  const maxChars = Number.isFinite(config.autoRecallMaxChars) && config.autoRecallMaxChars > 0 ? Math.trunc(config.autoRecallMaxChars) : 700;
+  const rawQuery = String(opts.query ?? "").trim().slice(0, 300);
+  const query = rawQuery.length > 0 ? rawQuery : (opts.allowRecencyFallback ?? true) ? null : "";
+  const items = query === null
+    ? (store.listMemories?.(workspace, limit) ?? []).slice(0, limit)
+    : store.recall(workspace, query, limit, { maxPerSession: limit });
+  const hits = Array.isArray(items) ? items.filter((m) => m !== null && m !== void 0) : [];
+  // "Few and precise" (agentmemory 候选①): never let a superseded stale row —
+  // which `recall` demotes to the tail of an unbounded list — occupy one of the
+  // ≤ N auto-injected slots. Same policy as the [ENGRAM] index. The stale row
+  // stays re-fetchable via engram_detail, just not auto-pushed.
+  const { supersededBy } = store.memoryRelations?.(workspace) ?? { supersededBy: new Map() };
+  const fresh = hits.filter((m) => !supersededBy.has(m.id));
+  if (fresh.length === 0) return "";
+
+  const lines = [];
+  const source = query === null ? "newest memory" : "recall";
+  lines.push(`[RECALL] ${source} · ${fresh.length} hit(s)${query ? ` · first msg: ${escapeLt(truncate(query, 40))}` : " · no user text yet"}`);
+  for (const m of fresh) {
+    const px = prefixFor(m.kind);
+    const mark = isProvenProcedure(m, config) ? `${px}✓` : px;
+    const date = fmtDate(m.createdAt ?? m.updatedAt);
+    const text = truncate(String(m.text).replace(/\s*\n+\s*/g, " ").trim(), 140);
+    lines.push(`- [${mark}] ${date} ${escapeLt(text)} #${shortId(m.id)}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
+  }
+  lines.push("context: use above · engram_detail <id> · engram_recall <query>");
+  return enforceStartRecallCap(lines, maxChars);
+}
+
+/** Keep the [RECALL] header + context tail, drop hits until under budget;
+ *  hard-truncate with an ellipsis when even the shell exceeds the cap. */
+function enforceStartRecallCap(lines, maxChars) {
+  let joined = lines.join("\n");
+  while (joined.length > maxChars && lines.length > 2) {
+    lines.splice(lines.length - 2, 1);
+    joined = lines.join("\n");
+  }
+  if (joined.length > maxChars) {
+    joined = `${joined.slice(0, Math.max(1, maxChars - 1))}…`;
+  }
+  return joined;
+}
+
 /**
  * One-line escalation reminder from an arbitrary balance view
  * ({ memCalls, esrCalls, days }). Surfaces the workspace real recent