dsh-engram 0.2.0

package/lib/index-block.js

@@ -0,0 +1,179 @@
+/**
+ * dsh-engram: the [ENGRAM] symbolic index + [ESR] task block.
+ *
+ * The whole token-saving philosophy lives here: instead of injecting recalled
+ * raw text, we inject ONE compact line per memory (type prefix · short date ·
+ * ≤56-char summary · short id) so the model knows what exists and can drill
+ * down on demand. The block is rendered once per session and then frozen (the
+ * section provider caches by session), keeping the prefix byte-stable for KV
+ * cache reuse.
+ */
+
+import { label, shortId, fmtDate, escapeLt, truncate } from "./util.js";
+
+/** Deterministic index line prefix per memory kind. */
+export function prefixFor(kind) {
+  switch (kind) {
+    case "decision": return "D";
+    case "error": return "E";
+    case "procedure": return "P";
+    case "insight": return "I";
+    case "handoff": return "H";
+    case "task": return "T";
+    default: return "F";
+  }
+}
+
+const INDEX_LINE_CHARS = 56;
+
+/**
+ * Git echoes captured before the plumbing filter still carry the full shell
+ * command ("git push: cd /repo && ...") — pure operational noise that should
+ * never steal a [ENGRAM] index line unless recall has proven it useful.
+ */
+function isGitPlumbingEcho(m) {
+  if (!Array.isArray(m.tags) || !m.tags.includes("auto-captured") || !m.tags.includes("git")) return false;
+  const text = String(m.text ?? "");
+  return /(^|\s)(cd\s|\&\&|;|\|)/.test(text);
+}
+
+/**
+ * Select the memories that earn an index line: knowingly stored high-signal
+ * entries, auto-captures past minIndexSignal, and low-signal entries promoted
+ * by repeated recall (hits >= promoteHits). Sorted newest first.
+ */
+export function selectIndexMemories(store, workspace, config) {
+  const all = store.listMemories(workspace, 200);
+  return all
+    .filter(
+      (m) =>
+        (isGitPlumbingEcho(m) && (m.hits ?? 0) < config.promoteHits
+          ? false
+          : (m.signal ?? 0.5) >= config.minIndexSignal ||
+            (m.hits ?? 0) >= config.promoteHits ||
+            m.kind === "task"),
+    )
+    .slice(0, config.indexMaxLines);
+}
+
+function memoryLine(m) {
+  const px = prefixFor(m.kind);
+  const date = fmtDate(m.createdAt ?? m.updatedAt);
+  const text = truncate(String(m.text).replace(/\s*\n+\s*/g, " ").trim(), INDEX_LINE_CHARS);
+  return `[${px}] ${date} ${escapeLt(text)} #${shortId(m.id)}`;
+}
+
+function taskStateLine(t) {
+  const gaps = [];
+  if (t.artifact === null || t.artifact === "") gaps.push("artifact");
+  if (t.evaluation === null || t.evaluation === "") gaps.push("evaluation");
+  if (t.memoryRefs.length === 0) gaps.push("memory_ref");
+  const state = t.state;
+  const suffix =
+    state === "stable"
+      ? "STABLE (artifact ✓ evaluation ✓ memory_ref ✓)"
+      : gaps.length === 0
+        ? "READY-to-close"
+        : `ACTIVE · gap: ${gaps.join(", ")}`;
+  const name = truncate(String(t.name).replace(/\s*\n+\s*/g, " ").trim(), INDEX_LINE_CHARS);
+  return `[T] ${fmtDate(t.createdAt)} ${name} — ${suffix} #${t.id.slice(0, 8)}`;
+}
+
+/**
+ * Render the full [ENGRAM] block (or "" when nothing to show). Pure function;
+ * called by the section provider and cached per session.
+ */
+export function renderIndex(store, workspace, cwd, config) {
+  const entries = store.workspaceEntries(workspace);
+  const memories = selectIndexMemories(store, workspace, config);
+  const tasks = store.listTasks(workspace, { includeStable: false });
+  const links = entries.links;
+
+  if (memories.length === 0 && tasks.length === 0 && links === 0) return "";
+
+  const lines = [];
+  lines.push(`[ENGRAM] workspace: ${label(cwd)} · ${entries.memories} memories · ${tasks.length} task(s) active · ${links} links · ${entries.nodes} node(s)`);
+  const entityNames = store.listEntities(workspace).map((e) => e.name);
+  if (entityNames.length > 0) lines.push(`nodes: ${truncate(entityNames.join(", "), 72)}`);
+  for (const m of memories.slice(0, config.indexMaxLines)) lines.push(memoryLine(m));
+  for (const t of tasks.slice(0, 4)) lines.push(taskStateLine(t));
+  lines.push("drill: engram_recall <query> | engram_detail <id> | esr_task (multi-step work → create now) | esr_node (recurring domain object → create now) | esr_link (relate nodes/tasks)");
+  return enforceCharCap(lines, config.indexMaxChars);
+}
+
+/**
+ * Keep the header + drill hint, drop the oldest content lines until the
+ * joined block fits the character budget (token discipline).
+ */
+function enforceCharCap(lines, maxChars) {
+  if (lines.length <= 2) return lines.join("\n");
+  let joined = lines.join("\n");
+  while (joined.length > maxChars && lines.length > 2) {
+    lines.splice(lines.length - 2, 1);
+    joined = lines.join("\n");
+  }
+  return joined;
+}
+
+/**
+ * One-line, data-driven escalation reminder. Surfaces the workspace real
+ * recent mem-vs-esr balance (usage rollup, last 14 days) so the model sees
+ * its own under-proactivity at the next session and can self-correct NOW.
+ * Returns "" when there is nothing to correct (no data, too small a sample,
+ * or the balance is healthy) — the reminder disappears once behaviour
+ * improves, closing the loop.
+ */
+export function escalationHint(store, workspace) {
+  const rows = store.usageRows(workspace);
+  if (rows.length === 0) return "";
+  const recent = rows.slice(-14);
+  const MEM = ["engram_store", "engram_recall", "engram_detail", "loom_store", "loom_recall", "loom_detail"];
+  const ESR = ["esr_task", "esr_node", "esr_close", "esr_link", "esr_gc"];
+  let memCalls = 0;
+  let esrCalls = 0;
+  for (const r of recent) {
+    const c = r.counts ?? {};
+    for (const k of MEM) memCalls += c[k] ?? 0;
+    for (const k of ESR) esrCalls += c[k] ?? 0;
+  }
+  // No data or too little signal — lecturing on <3 memory ops is noise.
+  if (memCalls < 3) return "";
+  const ratio = esrCalls / (memCalls + esrCalls);
+  // Healthy once esr exceeds ~half the memory-ops volume; otherwise nudge.
+  if (ratio >= 0.34) return "";
+  return `escalate: last ${recent.length}d ${memCalls} mem ops vs ${esrCalls} esr calls (${Math.round(ratio * 100)}%) — multi-step work → esr_task now, recurring objects → esr_node, related things → esr_link, and close with evidence (esr_close)`;
+}
+
+/**
+ * Render the [ESR] block — active tasks with closure gaps. Only appears when
+ * at least one task exists. "STABLE" tasks are folded to a count line to save
+ * tokens.
+ */
+export function renderEsr(store, workspace, config) {
+  const active = store.listTasks(workspace, { includeStable: false });
+  const stable = store.listTasks(workspace, { includeStable: true }).filter((t) => t.state === "stable");
+  const hint = escalationHint(store, workspace);
+  if (active.length === 0 && stable.length === 0) {
+    // Keep the mechanism visible even when idle: one compact line teaches the
+    // model that multi-step work earns an esr_task closed with real evidence.
+    const base = "[ESR] no open tasks — BE PROACTIVE: multi-step work gets a task now (esr_task); recurring domain objects get a node (esr_node); related things get a link (esr_link). Close tasks with real evidence (esr_close)";
+    return hint === "" ? base : `${base}\n${hint}`;
+  }
+
+  const lines = [];
+  lines.push(`[ESR] tasks: ${active.length} active / ${stable.length} stable`);
+  for (const t of active.slice(0, 8)) lines.push(`- ${t.id.slice(0, 6)}: ${escapeLt(truncate(t.name, 60))} — ${t.state.toUpperCase()}${taskGapSuffix(t)}`);
+  if (active.length > 8) lines.push(`- … ${active.length - 8} more`);
+  if (stable.length > 0) lines.push(`- closed: ${stable.slice(0, 6).map((t) => `${t.id.slice(0, 6)} (${truncate(t.name, 28)})`).join(" · ")}${stable.length > 6 ? ` +${stable.length - 6}` : ""}`);
+  if (hint !== "") lines.push(hint);
+  return lines.join("\n");
+}
+
+function taskGapSuffix(t) {
+  if (t.state === "stable") return "";
+  const gaps = [];
+  if (t.artifact === null || t.artifact === "") gaps.push("artifact");
+  if (t.evaluation === null || t.evaluation === "") gaps.push("evaluation");
+  if (t.memoryRefs.length === 0) gaps.push("memory_ref");
+  return gaps.length === 0 ? " · READY to close" : ` · gap: ${gaps.join(", ")}`;
+}

package/lib/index.js

@@ -0,0 +1,322 @@
+/**
+ * dsh-engram — minimalist symbolic-index × pi-esr memory for DeepSeek Harness.
+ *
+ * Host-half plugin only, no browser half, no external servers.
+ *   - zero-LLM auto-capture from tool results (tools/result)
+ *   - a compact, per-session-frozen [ENGRAM] symbolic index injected through a
+ *     systemPrompt section (order 40) — prefix-stable for KV cache reuse
+ *   - an [ESR] task/closure block (order 41) listing evidence gaps
+ *   - six tight tools: engram_store / engram_recall / engram_detail / esr_task /
+ *     esr_close / esr_link
+ *   - storage on ctx.storageDomain (JSON unit dsh_engram), reads synchronous
+ *
+ * All model-visible writes are immediate (no approval gate — zero friction,
+ * fully auditable through the session log provenance). Reads never call a
+ * model. The only optional model-free dependency is ctx.sessionQuery for the
+ * `search_sessions` FTS fallback.
+ */
+
+import { resolve as resolvePath } from "node:path";
+import { openEngramDomain } from "./store.js";
+import { registerTools } from "./tools.js";
+import { makeCaptureHandler } from "./capture.js";
+import { renderIndex, renderEsr } from "./index-block.js";
+import { installEngramSettings } from "./settings.js";
+import { makeEngramRoutes } from "./api.js";
+
+/** Stable cordis plugin name. */
+export const name = "engram";
+
+/** Tools must exist before any registration. */
+export const inject = ["tools"];
+
+/** Defaults; every key can be overridden via profile patch config. */
+const DEFAULTS = {
+  enabled: true,
+  // intake
+  autoCapture: true,
+  autoCapturePerSession: 40,
+  autoCaptureGlobalCap: 500,
+  // recall
+  sessionSearch: true,
+  // [ENGRAM] index budget (token discipline)
+  indexMaxLines: 12,
+  indexMaxChars: 700,
+  minIndexSignal: 0.4,
+  promoteHits: 3,
+  // retention / caps
+  expireDays: 180,
+  maxMemoriesPerWorkspace: 2000,
+  maxMemoryChars: 1600,
+  maxTasksPerWorkspace: 40,
+  // memory GC (pi-esr constraints, archive-only)
+  gcEnabled: true,
+  gcIntervalHours: 24,
+  gcStableRetentionDays: 120,
+  // GUI access: hostnames allowed past the loopback fence for /api/dsh-engram
+  // (e.g. an operator-authorized cloudflare tunnel). Empty = loopback-only.
+  trustedHosts: [],
+  // prompt section order (before the persona/tools bands per DSH convention)
+  engramIndexOrder: 40,
+  esrOrder: 41,
+};
+
+function resolveConfig(config) {
+  const src = config && typeof config === "object" ? config : {};
+  const out = { ...DEFAULTS };
+  for (const key of Object.keys(DEFAULTS)) {
+    if (src[key] !== void 0) out[key] = src[key];
+  }
+  return out;
+}
+
+/**
+ * Mount the plugin. `ctx` is the scoped cordis context; `config` the optional
+ * resolved plugin config (plain object from the profile patch).
+ */
+export function apply(ctx, config) {
+  const resolved = resolveConfig(config);
+  if (!resolved.enabled) return;
+
+  const log = ctx.logger ? ctx.logger("engram") : undefined;
+  const sessionQuery = ctx.get("sessionQuery");
+
+  // `storageDomain` is provided asynchronously by the storage-domain row —
+  // its apply awaits a json backend via ctx.inject before providing the
+  // facility, and the loader starts rows concurrently. Snapshotting it here
+  // freezes `undefined` forever when the provide lands right after this
+  // apply (consistently the case on slower machines like Windows). Resolve it
+  // lazily through cordis's inject instead, with a bounded fallback so a
+  // genuinely facility-less profile still surfaces the friendly disabled
+  // message rather than hanging every request.
+  const storageDomainPromise = new Promise((resolve, reject) => {
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      const current = ctx.get("storageDomain");
+      if (current !== void 0) resolve(current);
+      else {
+        reject(
+          new Error("the storage-domain facility is not available in this profile — engram memory is disabled here"),
+        );
+      }
+    }, 10_000);
+    ctx.inject(["storageDomain"], (scoped) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(scoped.get("storageDomain") ?? ctx.get("storageDomain"));
+    });
+  });
+
+  let domainPromise;
+  let opened;
+  const getDomain = () => {
+    domainPromise ??= storageDomainPromise.then((storageDomain) => openEngramDomain(storageDomain)).then((domain) => {
+      opened = domain;
+      return domain;
+    });
+    return domainPromise;
+  };
+
+  const service = {
+    config: resolved,
+    getDomain,
+    openedDomain: () => opened,
+    ensureDomain: () => {
+      getDomain().catch(() => {});
+    },
+    sessionQuery,
+    log,
+    /** Cumulative auto-capture totals (GUI overview observability). */
+    captureStats: { total: 0, git: 0, file: 0, error: 0 },
+    /** Last GC sweep totals (GUI overview observability). */
+    gcStats: { lastRun: 0, archivedMemories: 0, archivedTasks: 0, removedLinks: 0 },
+    /** Recompute the exact [ENGRAM] block for a workspace (GUI overview cost). */
+    renderIndexBlock(workspace) {
+      if (opened === void 0) return "";
+      try {
+        return renderIndex(opened, workspace, workspace, resolved);
+      } catch {
+        return "";
+      }
+    },
+  };
+
+  // Mechanical, archive-only memory GC (pi-esr constraints). Runs the first
+  // sweep once the store is reachable, then on the configured interval. Every
+  // entry archived stays re-fetchable — nothing is hard-deleted by GC.
+  const runGcAll = async () => {
+    if (!resolved.gcEnabled) return;
+    try {
+      const domain = await getDomain();
+      const report = await domain.gc(void 0, resolved, { dryRun: false });
+      service.gcStats.lastRun = Date.now();
+      service.gcStats.archivedMemories += report.archivedMemories.length;
+      service.gcStats.archivedTasks += report.archivedTasks.length;
+      service.gcStats.removedLinks += report.removedLinks.length;
+      if (report.archivedMemories.length + report.archivedTasks.length + report.removedLinks.length > 0) {
+        log?.info?.(`engram gc: ${report.archivedMemories.length} memories archived, ${report.archivedTasks.length} tasks, ${report.removedLinks.length} links removed`);
+      }
+    } catch (error) {
+      log?.warn?.(`engram gc sweep failed: ${String(error)}`);
+    }
+  };
+
+  // User settings (web GUI card): merges stored values onto the live config.
+  // Non-blocking — when the settings service is absent this never runs.
+  installEngramSettings(ctx, resolved, resolved);
+
+  // Web API for the GUI: deferred on the webServer service, so headless
+  // profiles keep running the plugin without it. `register` returns the
+  // disposer immediately (the dsh-ssh family does the same).
+  ctx.inject(["webServer"], (webCtx) => {
+    const routes = makeEngramRoutes(service);
+    const disposers = [];
+    for (const route of routes) {
+      try {
+        disposers.push(webCtx.webServer.register(route));
+      } catch {}
+    }
+    return () => {
+      for (const dispose of disposers) {
+        try {
+          dispose();
+        } catch {}
+      }
+    };
+  });
+
+  // Per-session frozen blocks: render once, then reuse — prefix stability is
+  // the point (KV cache). New session (or resumed reload) object → fresh block.
+  const sessionCache = new WeakMap();
+  const blockFor = (agent) => {
+    if (agent === void 0) return "";
+    const session = agent.session;
+    if (session === void 0) return "";
+    const cwd = session.header?.cwd;
+    if (cwd === void 0 || cwd.length === 0) return "";
+    const cached = sessionCache.get(session);
+    if (cached !== void 0) return cached;
+    if (opened === void 0) {
+      service.ensureDomain();
+      return "";
+    }
+    let block = "";
+    try {
+      const workspace = resolvePath(cwd);
+      block = renderIndex(opened, workspace, cwd, resolved);
+    } catch (error) {
+      log?.warn?.(`engram index render failed: ${String(error)}`);
+    }
+    sessionCache.set(session, block);
+    return block;
+  };
+
+  ctx.effect(() => {
+    const disposers = [];
+
+    // tools
+    disposers.push(registerTools(ctx, service));
+
+    // systemPrompt sections (frozen per session)
+    const systemPrompt = ctx.get("systemPrompt");
+    if (systemPrompt !== void 0) {
+      const indexDispose = systemPrompt.section({
+        name: "engram:index",
+        order: resolved.engramIndexOrder,
+        text: (assemblyContext) => {
+          const agent = assemblyContext?.agent;
+          if (agent === void 0) return "";
+          const session = agent.session;
+          if (session === void 0) return "";
+          const cwd = session.header?.cwd;
+          if (cwd === void 0 || cwd.length === 0) return "";
+          return blockFor(agent);
+        },
+      });
+      disposers.push(() => indexDispose());
+
+      const esrDispose = systemPrompt.section({
+        name: "engram:esr",
+        order: resolved.esrOrder,
+        text: (assemblyContext) => {
+          const agent = assemblyContext?.agent;
+          if (agent === void 0) return "";
+          const session = agent.session;
+          if (session === void 0) return "";
+          const cwd = session.header?.cwd;
+          if (cwd === void 0 || cwd.length === 0) return "";
+          if (opened === void 0) return "";
+          try {
+            return renderEsr(opened, resolvePath(cwd), resolved);
+          } catch {
+            return "";
+          }
+        },
+      });
+      disposers.push(() => esrDispose());
+    }
+
+    // zero-LLM auto-capture
+    if (resolved.autoCapture) {
+      // The capture handler needs a storeMemory face, but the domain opens
+      // lazily. The adapter resolves it per call and drops events that arrive
+      // before the first open (rate buckets still counted, so nothing is lost
+      // once the domain is up).
+      const captureStore = {
+        storeMemory(...args) {
+          const domain = service.openedDomain();
+          if (domain === void 0) {
+            service.ensureDomain();
+            return Promise.reject(new Error("engram domain not open yet"));
+          }
+          return domain.storeMemory(...args);
+        },
+      };
+      const capture = makeCaptureHandler(captureStore, resolved, log, service.captureStats);
+      const disposeCapture = ctx.on("tools/result", (exec, result) => {
+        capture(exec, result);
+      });
+      disposers.push(disposeCapture);
+    }
+
+    // memory GC scheduling: one sweep after the store is reachable, then on
+    // the configured interval (cordis clears ctx-set timers on dispose).
+    // Guard against a boot-order race: ctx.setInterval routes through the
+    // `timer` service, which may not be injected yet at apply time, and an
+    // unguarded read crashes the whole server (`cannot get property "timer"
+    // without inject`). Fall back to a global interval; GC is best-effort.
+    getDomain()
+      .then(() => {
+        setTimeout(() => void runGcAll(), 0);
+      })
+      .catch(() => {});
+    const gcMs = Math.max(1, resolved.gcIntervalHours) * 60 * 60 * 1000;
+    let gcDispose;
+    try {
+      gcDispose = ctx.setInterval(() => void runGcAll(), gcMs);
+    } catch {
+      gcDispose = undefined;
+    }
+    if (gcDispose === undefined) {
+      const gcHandle = globalThis.setInterval(() => void runGcAll(), gcMs);
+      gcDispose = () => globalThis.clearInterval(gcHandle);
+    }
+    disposers.push(gcDispose);
+
+    return async () => {
+      for (const dispose of disposers) {
+        try {
+          dispose();
+        } catch {}
+      }
+      if (domainPromise !== void 0) {
+        try {
+          await (await domainPromise).close();
+        } catch {}
+      }
+    };
+  });
+}

package/lib/settings.js

@@ -0,0 +1,95 @@
+/**
+ * dsh-engram: user settings binding (host half).
+ *
+ * Registers the `dsh-engram` settings namespace through DSH's own settings
+ * service so the web GUI's Plugins configuration tab pairs the namespace with
+ * the plugin's card automatically. Changes land live on `service.config`
+ * (new sessions pick them up; blocks already frozen for a session stay frozen
+ * by design — prefix stability wins over mid-session knobs).
+ *
+ * The schema is schemastery (DSH's own schema engine), not zod: the settings
+ * transport and the form renderers are all schemastery-native.
+ */
+
+import z from "@deepseek-ai/schemastery";
+import { settingsNamespace, installSettingsSection } from "@deepseek-ai/dsh-settings";
+
+/** Namespace join key — must match the client card's `key`. */
+export const SETTINGS_NS_NAME = "dsh-engram";
+export const SETTINGS_NS = settingsNamespace(SETTINGS_NS_NAME);
+
+/** Fields the GUI exposes (a curated subset of the plugin DEFAULTS). */
+export const SETTINGS_KEYS = [
+  "autoCapture",
+  "sessionSearch",
+  "autoCapturePerSession",
+  "indexMaxLines",
+  "indexMaxChars",
+  "minIndexSignal",
+  "promoteHits",
+  "expireDays",
+  "maxMemoriesPerWorkspace",
+  "gcEnabled",
+  "gcStableRetentionDays",
+  "trustedHosts",
+];
+
+/** Base (composition) layer for the settings section, from plugin defaults. */
+export function settingsBaseFrom(defaults) {
+  const base = {};
+  for (const key of SETTINGS_KEYS) {
+    if (defaults[key] !== void 0) base[key] = defaults[key];
+  }
+  return base;
+}
+
+/** schemastery schema describing the GUI-editable fields. */
+export function makeSettingsSchema(defaults) {
+  return z.object({
+    autoCapture: z.boolean().default(defaults.autoCapture),
+    sessionSearch: z.boolean().default(defaults.sessionSearch),
+    autoCapturePerSession: z.natural().max(1000).default(defaults.autoCapturePerSession),
+    indexMaxLines: z.natural().max(50).default(defaults.indexMaxLines),
+    indexMaxChars: z.natural().max(4000).default(defaults.indexMaxChars),
+    minIndexSignal: z.percent().default(defaults.minIndexSignal),
+    promoteHits: z.natural().max(20).default(defaults.promoteHits),
+    expireDays: z.natural().max(3650).default(defaults.expireDays),
+    maxMemoriesPerWorkspace: z.natural().max(10000).default(defaults.maxMemoriesPerWorkspace),
+    gcEnabled: z.boolean().default(defaults.gcEnabled),
+    gcStableRetentionDays: z.natural().max(3650).default(defaults.gcStableRetentionDays),
+    // Security knob, not rendered in the card: hostnames allowed past the
+    // loopback fence for /api/dsh-engram (operator-authorized tunnel hosts).
+    trustedHosts: z.array(z.string()).default([]),
+  });
+}
+
+/**
+ * Wire the settings section into `ctx` and keep `live` (the plugin's mutable
+ * config object) merged with what the user stored. Called from `apply()`.
+ */
+export function installEngramSettings(ctx, defaults, live) {
+  const base = settingsBaseFrom(defaults);
+  const schema = makeSettingsSchema(defaults);
+  let source = () => base;
+  installSettingsSection(ctx, SETTINGS_NS, schema, base, {
+    setSource(current) {
+      source = current;
+    },
+    onChange() {
+      let value;
+      try {
+        value = source();
+      } catch {
+        return;
+      }
+      if (value === null || typeof value !== "object") return;
+      for (const key of SETTINGS_KEYS) {
+        if (value[key] !== void 0 && live[key] !== value[key]) live[key] = value[key];
+      }
+    },
+    validate(value) {
+      // The schema already constrains every exposed field; refuse nothing here.
+      void value;
+    },
+  });
+}