dsh-engram 0.4.0 → 0.5.1

package/lib/index-block.js

@@ -20,6 +20,7 @@ export function prefixFor(kind) {
     case "insight": return "I";
     case "handoff": return "H";
     case "task": return "T";
+    case "preference": return "R";
     default: return "F";
   }
 }
@@ -65,14 +66,24 @@ export function selectIndexMemories(store, workspace, config) {
             m.kind === "task"),
     )
     .sort((a, b) => {
-      const ap = isProvenProcedure(a, config) ? 0 : 1;
-      const bp = isProvenProcedure(b, config) ? 0 : 1;
+      // User preferences are standing rules — rank them above everything
+      // (still newest-first within the group). Proven procedures follow.
+      const ap = kindRank(a, config) ? 0 : 1;
+      const bp = kindRank(b, config) ? 0 : 1;
       if (ap !== bp) return ap - bp;
+      const ap2 = isProvenProcedure(a, config) ? 0 : 1;
+      const bp2 = isProvenProcedure(b, config) ? 0 : 1;
+      if (ap2 !== bp2) return ap2 - bp2;
       return byRecency(a, b);
     })
     .slice(0, config.indexMaxLines);
 }
 
+/** Preference memories outrank everything in the [ENGRAM] index block. */
+function kindRank(m, config) {
+  return (m?.kind ?? "") === "preference" && (m.signal ?? 0.5) >= (config?.minIndexSignal ?? 0.4);
+}
+
 function memoryLine(m, config) {
   const px = prefixFor(m.kind);
   const mark = isProvenProcedure(m, config) ? `${px}✓` : px;
@@ -199,14 +210,25 @@ export function renderStartRecall(store, workspace, config = {}, opts = {}) {
   const lines = [];
   const source = query === null ? "newest memory" : "recall";
   lines.push(`[RECALL] ${source} · ${fresh.length} hit(s)${query ? ` · first msg: ${escapeLt(truncate(query, 40))}` : " · no user text yet"}`);
+  // UAP (read-path activation): when a hit's oneline is truncated, say so —
+  // the model should know the full record is one `engram_detail` call away
+  // instead of assuming the 140-char preview is the whole memory. The trailing
+  // guidance then points at a concrete truncated id (or the top hit).
+  let truncatedId = null;
   for (const m of fresh) {
     const px = prefixFor(m.kind);
     const mark = isProvenProcedure(m, config) ? `${px}✓` : px;
     const date = fmtDate(m.createdAt ?? m.updatedAt);
-    const text = truncate(String(m.text).replace(/\s*\n+\s*/g, " ").trim(), 140);
-    lines.push(`- [${mark}] ${date} ${escapeLt(text)} #${shortId(m.id)}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
+    const full = String(m.text).replace(/\s*\n+\s*/g, " ").trim();
+    const truncated = [...full].length > 140;
+    const text = truncate(full, 140);
+    if (truncated && truncatedId === null) truncatedId = shortId(m.id);
+    lines.push(`- [${mark}] ${date} ${escapeLt(text)} #${shortId(m.id)}${truncated ? " …" : ""}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
   }
-  lines.push("context: use above · engram_detail <id> · engram_recall <query>");
+  const drill = truncatedId !== null
+    ? `context: use above · #${truncatedId} truncated — engram_detail ${truncatedId} (full) · engram_recall <query>`
+    : `context: use above · engram_detail <id> · engram_recall <query>`;
+  lines.push(drill);
   return enforceStartRecallCap(lines, maxChars);
 }
 

package/lib/index.js

@@ -23,11 +23,14 @@ import { openEngramDomain } from "./store.js";
 import { registerTools } from "./tools.js";
 import { makeCaptureHandler } from "./capture.js";
 import { installGoalCapture } from "./goal-capture.js";
+import { installPreferenceCapture } from "./preference.js";
 import { installBoundaryPrune } from "./boundary-prune.js";
+import { resultPackDir, archivePrunedResults, resultRecallTool, sweepExpired } from "./result-pack.js";
+import { recordBoundaryPruneFire, recordRecallInjection, readPackTelemetry, engramHome } from "./pack-telemetry.js";
 import { renderIndex, renderStartRecall, firstUserMessageText, renderEsr, esrMethodology, buildEsrSnapshot, buildFrozenEsrBlock } from "./index-block.js";
 import { makeTriggerRecorder } from "./trigger.js";
 import { pendingTodosFromSession, sinkPendingTodos } from "./todo-sink.js";
-import { installEngramSettings } from "./settings.js";
+import { installEngramSettings, makeEngineConfigSchema, SETTINGS_KEYS } from "./settings.js";
 import { makeEngramRoutes } from "./api.js";
 import { collectPointerContext, mountCompactionEngine } from "./context-gc.js";
 import { provisionWebCompaction } from "./web-provision.js";
@@ -68,6 +71,11 @@ const DEFAULTS = {
   autoRecallOnStart: true,
   autoRecallLimit: 3,
   autoRecallMaxChars: 700,
+  // user-correction / preference learning (zero-LLM, auto-captured from
+  // user/message events; prefer/avoid/switch-to cues). Store as `preference`
+  // memories that outrank the [ENGRAM] index.
+  autoCapturePreferences: true,
+  preferenceCapturePerSession: 8,
   // [ENGRAM] index budget (token discipline)
   indexMaxLines: 12,
   indexMaxChars: 700,
@@ -144,23 +152,110 @@ const DEFAULTS = {
   // least this many more requests (empirical workspace prior, see
   // lib/boundary-prune.js). 50 = L0 sweep optimum.
   boundaryPruneMinRemaining: 50,
+  // Threshold override (ResultPack L0 "T2048 landing"): the host pruner is
+  // fixed at 8192; set this to a lower budget to prune earlier at boundaries
+  // (L0 says T2048 has the highest net savings +61.6Mc, gated only by the
+  // L2 archive requirement — now satisfied). 0/absent → host threshold.
+  boundaryPruneThresholdChars: 0,
+  // Edges retained by the override pruner (unless set, host's 4096/1024).
+  boundaryPruneHeadChars: 0,
+  boundaryPruneTailChars: 0,
+  // Request-count boundary (SoL-Pi "step idle", default OFF): fire at most once
+  // per N requests even in sessions with no todo/goal boundary (~62% of the L0
+  // corpus never fires the event boundaries). Still under the repayment gate.
+  // 0 → event boundaries only.
+  boundaryPruneIdleCadence: 0,
+  // ResultPack L2 (PROPOSAL-resultpack §L2): archive the middles L1 prunes
+  // off the surface (~/.dsh/engram/resultpack/<callId>.json, redacted) and
+  // append a `[middle archived: result_recall(<callId>) 可取]` handle tail to
+  // each pruned node; register `result_recall(callId, offset?)` for paged
+  // recall — the full ObservationPack pattern (SoL-Pi mech ③; paired
+  // EdgeBench −23.58%). Requires `boundaryPrune: true` to have anything to
+  // archive (L2 without L1 is a no-op with a log line). Default OFF (C13).
+  resultPack: false,
+  // result_recall page budget (chars per page). 4000 fits a typical model
+  // context band without re-materializing huge results whole.
+  resultPackPageChars: 4000,
+  // Retain archived middles this many days (mtime sweep, same GC cadence).
+  resultPackRetentionDays: 7,
 };
 
+/**
+ * Cordis reads `plugin.Config` off the module: DSH ≥0.1.7 validates the
+ * `engram` profile row against it AND projects the settings form from it —
+ * the curated SETTINGS_KEYS are `.volatile()`, so the settings service exposes
+ * exactly the GUI card's knobs (entry id `engram`), while every other DEFAULTS
+ * key stays a plain declared field so profile-patch config survives validation.
+ */
+export const Config = makeEngineConfigSchema(DEFAULTS);
+
 export function resolveConfig(config) {
   const src = config && typeof config === "object" ? config : {};
   const out = { ...DEFAULTS };
   for (const key of Object.keys(DEFAULTS)) {
-    if (src[key] !== void 0) out[key] = src[key];
+    if (src[key] !== void 0) {
+      // A volatile reference may hold a nullish value (e.g. a profile patch
+      // listed a knob as null, or schemastery's null-default sentinel); the
+      // `??` keeps the DEFAULTS value in that case instead of nulling out.
+      out[key] = unwrapValue(src[key]) ?? DEFAULTS[key];
+    }
   }
   return out;
 }
 
+/** Unwrap a volatile config reference (DSH ≥0.1.7 settings) to its plain value. */
+function unwrapValue(value) {
+  return value !== null && typeof value === "object" && typeof value.get === "function" ? value.get() : value;
+}
+
+/**
+ * Collect the volatile config references for the given keys from a plugin
+ * config object (DSH ≥0.1.7 validates `Config` into volatile references that
+ * the loader commits into IN PLACE on settings-card edits). Plain values are
+ * skipped, so this is a no-op on legacy DSH.
+ */
+export function collectVolatileRefs(config, keys) {
+  const refs = {};
+  if (config !== null && typeof config === "object") {
+    for (const key of keys) {
+      const ref = config[key];
+      if (ref !== null && typeof ref === "object" && typeof ref.get === "function") refs[key] = ref;
+    }
+  }
+  return refs;
+}
+
+/**
+ * Re-read volatile references into a resolved config (`target`, mutated in
+ * place): the loader has committed a settings-card edit into the same refs and
+ * emitted `loader/volatile-update`. Nullish refs fall back to DEFAULTS (the
+ * schemastery null-default sentinel yields an undefined ref value). Returns
+ * the keys whose value actually changed.
+ */
+export function syncVolatileRefs(refs, target) {
+  const changed = [];
+  for (const key of Object.keys(refs)) {
+    const next = unwrapValue(refs[key]) ?? DEFAULTS[key];
+    if (next !== target[key]) {
+      target[key] = next;
+      changed.push(key);
+    }
+  }
+  return changed;
+}
+
 /**
  * Mount the plugin. `ctx` is the scoped cordis context; `config` the optional
  * resolved plugin config (plain object from the profile patch).
  */
 export function apply(ctx, config) {
   const resolved = resolveConfig(config);
+  // DSH ≥0.1.7 / 0.2.x settings model: the GUI card edits the `engram`
+  // entry's VOLATILE Config fields. The loader commits those into the same
+  // references (in place) and emits `loader/volatile-update` — hold the refs
+  // so a card edit re-syncs `resolved` live for new sessions (blocks already
+  // frozen for a session stay frozen — the plugin's documented contract).
+  const volatileRefs = collectVolatileRefs(config, SETTINGS_KEYS);
   if (!resolved.enabled) return;
 
   const log = ctx.logger ? ctx.logger("engram") : undefined;
@@ -220,7 +315,10 @@ export function apply(ctx, config) {
     sessionQuery,
     log,
     /** Cumulative auto-capture totals (GUI overview observability). */
-    captureStats: { total: 0, git: 0, file: 0, error: 0 },
+    captureStats: { total: 0, git: 0, file: 0, error: 0, preference: 0 },
+    /** SoL-Pi activation telemetry (M15/C13): durable boundary-prune fires +
+     *  [RECALL] injection funnel, read at request time (readPackTelemetry). */
+    packTelemetry: () => readPackTelemetry({ home: engramHome() }),
     /** Last GC sweep totals (GUI overview observability). */
     gcStats: { lastRun: 0, archivedMemories: 0, archivedTasks: 0, removedLinks: 0 },
     /** Live Context GC status (API overview + durable status file). */
@@ -264,6 +362,11 @@ export function apply(ctx, config) {
       if (report.archivedMemories.length + report.archivedTasks.length + report.removedLinks.length > 0) {
         log?.info?.(`engram gc: ${report.archivedMemories.length} memories archived, ${report.archivedTasks.length} tasks, ${report.removedLinks.length} links removed`);
       }
+      // ResultPack L2 archive sweep rides the same GC cadence (mtime-based).
+      if (resolved.resultPack === true) {
+        const removed = sweepExpired(resultPackDir(), resolved.resultPackRetentionDays ?? 7);
+        if (removed > 0) log?.info?.(`engram gc: ${removed} expired result-pack archives removed`);
+      }
     } catch (error) {
       log?.warn?.(`engram gc sweep failed: ${String(error)}`);
     }
@@ -327,7 +430,18 @@ export function apply(ctx, config) {
           pendingRecall = true;
         } else {
           const recall = renderStartRecall(opened, workspace, resolved, { query, allowRecencyFallback: false });
-          block = recall.length > 0 ? `${block}\n${recall}`.trim() : block;
+          if (recall.length > 0) {
+            // Activation funnel, injection leg: a [RECALL] block actually made
+            // it into a session. Counted here (the block only materializes when
+            // the render produced hits), so /stats can close the loop with
+            // engram_recall/engram_detail tool calls from the session logs.
+            recordRecallInjection({
+              home: engramHome(),
+              workspace,
+              hits: (recall.match(/- \[/g) ?? []).length,
+            });
+            block = `${block}\n${recall}`.trim();
+          }
         }
       }
     } catch (error) {
@@ -392,6 +506,17 @@ export function apply(ctx, config) {
   ctx.effect(() => {
     const disposers = [];
 
+    // Settings card edits (volatile Config fields) hot-commit into the refs
+    // in place; re-read them so `resolved` reflects new values for new
+    // sessions without a profile restart. Legacy DSH never emits this event
+    // and leaves `volatileRefs` empty, so the listener is inert there.
+    if (Object.keys(volatileRefs).length > 0) {
+      const disposeVolatile = ctx.on?.("loader/volatile-update", () => {
+        syncVolatileRefs(volatileRefs, resolved);
+      });
+      if (disposeVolatile !== void 0) disposers.push(disposeVolatile);
+    }
+
     // tools
     disposers.push(registerTools(ctx, service));
 
@@ -622,11 +747,62 @@ export function apply(ctx, config) {
       const disposeGoal = installGoalCapture(ctx, captureStore, log);
       disposers.push(disposeGoal);
 
+      // User-correction / preference learning (zero-LLM): `user/message`
+      // events carrying explicit prefer/avoid/switch-to cues auto-sediment as
+      // `preference` memories (correx / PAHF / Mem0 preference-extraction
+      // pattern, distilled to pure rules). No model call.
+      const disposePreference = installPreferenceCapture(ctx, captureStore, resolved, log, service.captureStats);
+      disposers.push(disposePreference);
+
       // BoundaryPrune (PROPOSAL-resultpack L1, default OFF): prune
       // over-threshold tool results at todo/goal subtask boundaries through
       // DSH's own deterministic toolResultPruner, gated by a repayment clock.
-      const disposeBoundaryPrune = installBoundaryPrune(ctx, resolved, log);
+      const l2Dir = resultPackDir();
+      const disposeBoundaryPrune = installBoundaryPrune(ctx, resolved, log, {
+        // ResultPack L2 hook: archive the pruned middles (redacted) + append
+        // a handle tail on each pruned node. Fail-contained in result-pack.js.
+        // Also records durable activation telemetry (SoL-Pi M15/C13) —
+        // independent of `resultPack`: /stats must be able to prove the L1
+        // mechanism actually fires on the live profile.
+        onPruned: (session, report, boundary) => {
+          recordBoundaryPruneFire({
+            home: engramHome(),
+            boundary,
+            report,
+            workspace: wk(session?.header?.cwd),
+          });
+          if (resolved.resultPack !== true) return;
+          const workspace = wk(session?.header?.cwd);
+          archivePrunedResults({
+            dir: l2Dir,
+            session,
+            report,
+            workspace,
+            log,
+          });
+        },
+      });
       disposers.push(disposeBoundaryPrune);
+
+      // ResultPack L2 tool (default OFF with L1): `result_recall(callId, offset?)`
+      // pages back an archived middle. Independent of the memory domain — the
+      // archive lives on the filesystem. Registers only when enabled; without
+      // boundayPrune there is nothing to archive, so it self-disables.
+      if (resolved.resultPack === true) {
+        if (resolved.boundaryPrune !== true) {
+          log?.warn?.("engram result-pack: requires boundaryPrune: true to prune middles — result_recall not registered");
+        } else {
+          const pageChars = Number.isFinite(resolved.resultPackPageChars)
+            ? resolved.resultPackPageChars
+            : 4000;
+          disposers.push(
+            ctx.effect(() => {
+              ctx.tools.register(resultRecallTool(l2Dir, pageChars));
+              log?.info?.(`engram result-pack enabled: result_recall(callId, offset) registered; archive dir ${l2Dir}`);
+            }),
+          );
+        }
+      }
     }
 
     // memory GC scheduling: one sweep after the store is reachable, then on

package/lib/pack-telemetry.js

@@ -0,0 +1,115 @@
+/**
+ * Activation telemetry for the SoL-Pi mechanisms (dsh-engram).
+ *
+ * SoL-Pi methodology pair rule: M15 "mechanisms must prove activation" and
+ * C13 "dormant mechanisms are off by default" — a mechanism that cannot be
+ * observed activating is treated as dormant. This module gives BoundaryPrune
+ * (L1) + ResultPack (L2) an observable, durable activation record:
+ *
+ *   - `recordBoundaryPruneFire` is called on every real prune (fires > 0) —
+ *     durable JSON under the engram home, cumulative across restarts, so
+ *     /stats and the GUI overview can prove the mechanism actually fires on
+ *     the live profile (not just in the simulation).
+ *   - `readBoundaryPruneStats` / `readRecallInjections` are pure readers.
+ *
+ * Zero LLM, pure deterministic bookkeeping. Two files today:
+ *   ~/.dsh/engram/boundary-prune.status.json
+ *   ~/.dsh/engram/recall-injections.json
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+/** Default engram home (mirrors result-pack.js). */
+export function engramHome(home = path.join(os.homedir(), ".dsh", "engram")) {
+  return home;
+}
+
+function statusFile(home, name) {
+  return path.join(home, name);
+}
+
+/** Atomic-ish write: tmp + rename, so a crash never leaves a torn file. */
+function writeJsonAtomic(file, payload) {
+  try {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const tmp = `${file}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(payload)}\n`, "utf8");
+    fs.renameSync(tmp, file);
+  } catch {
+    // best-effort telemetry — never throw into the prune pipeline
+  }
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Record one BoundaryPrune fire (a real prune happened) into durable state.
+ * `report` is the pruner's pruneSession result. Returns the new cumulative
+ * stats object (or null when writing failed).
+ */
+export function recordBoundaryPruneFire({ home, boundary, report, workspace, now = Date.now() } = {}) {
+  const prev = readBoundaryPruneStats({ home }) ?? {};
+  const pruned = Array.isArray(report?.pruned) ? report.pruned.length : 0;
+  const chars = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
+  const kind = boundary?.kind ?? "unknown";
+  const byKind = { ...(prev.byKind ?? {}) };
+  byKind[kind] = (byKind[kind] ?? 0) + 1;
+  const next = {
+    version: 1,
+    fires: (prev.fires ?? 0) + 1,
+    prunedResults: (prev.prunedResults ?? 0) + pruned,
+    charsRemoved: (prev.charsRemoved ?? 0) + chars,
+    uses: (prev.uses ?? 0) + 1, // legacy alias kept for GUI tolerance
+    lastFire: now,
+    lastBoundary: kind,
+    lastWorkspace: workspace ?? prev.lastWorkspace ?? null,
+    byKind,
+  };
+  writeJsonAtomic(statusFile(home, "boundary-prune.status.json"), next);
+  return next;
+}
+
+/** Read the cumulative BoundaryPrune activation record (never throws). */
+export function readBoundaryPruneStats({ home } = {}) {
+  return readJson(statusFile(home, "boundary-prune.status.json"));
+}
+
+/**
+ * Record one [RECALL] injection (a session-start recall block was actually
+ * appended to a session) for the activation funnel: injections → tool
+ * queries → hits → detail-follows.
+ */
+export function recordRecallInjection({ home, workspace, hits, now = Date.now() } = {}) {
+  const prev = readRecallInjections({ home }) ?? { version: 1, injections: 0, hitInjections: 0 };
+  const next = {
+    version: 1,
+    injections: (prev.injections ?? 0) + 1,
+    hitInjections: (prev.hitInjections ?? 0) + (hits > 0 ? 1 : 0),
+    hitsInjected: (prev.hitsInjected ?? 0) + (hits > 0 ? hits : 0),
+    lastInjection: now,
+    lastWorkspace: workspace ?? prev.lastWorkspace ?? null,
+  };
+  writeJsonAtomic(statusFile(home, "recall-injections.json"), next);
+  return next;
+}
+
+/** Read the cumulative [RECALL] injection record (never throws). */
+export function readRecallInjections({ home } = {}) {
+  return readJson(statusFile(home, "recall-injections.json"));
+}
+
+/** Aggregate both telemetry files into one summary for /stats + overview. */
+export function readPackTelemetry({ home } = {}) {
+  return {
+    boundaryPrune: readBoundaryPruneStats({ home }),
+    recall: readRecallInjections({ home }),
+  };
+}