dsh-engram 0.7.4 → 0.8.7

package/lib/skill-export.js

@@ -0,0 +1,289 @@
+/**
+ * v0.8-B: export evidence-backed procedures as standard DSH Skills.
+ *
+ * Why this exists: the [ENGRAM] index line is a prompt hint — it disappears when
+ * a session compacts and it never leaves this machine's memory store. A DSH
+ * Skill is a different surface: the host discovers it from `.dsh/skills`,
+ * advertises it in the session catalog, and any agent (or subagent) can load the
+ * FULL text on demand. So a procedure that has proven itself 3 times deserves to
+ * exist as a real artifact, not just a line of recalled text.
+ *
+ * The contract with the host is verified, not assumed
+ * (`@deepseek-ai/dsh-skill-filesystem`):
+ *   - roots: `<project>/.dsh/skills` (rank 100), `<project>/.agents/skills`,
+ *     `$DSH_HOME/skills`, `$AGENTS_HOME/skills`;
+ *   - a skill is a DIRECTORY containing `SKILL.md` (or a flat `*.md`);
+ *   - frontmatter MUST carry `name` and `description`, `name` matching
+ *     /^[a-z0-9]+(?:-[a-z0-9]+)*$/; `whenToUse` and `disable-model-invocation` /
+ *     `user-invocable` are optional.
+ *
+ * Two safety rules, because this module writes into a directory the host reads:
+ *   1. only files carrying SKILL_MARKER are ever rewritten or deleted — a
+ *      hand-written skill in the same root is never touched;
+ *   2. nothing is exported until it passes the v0.8-A evidence gate
+ *      (≥3 verified successes, verified within 14 days, not invalidated).
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+import { isProvenProcedure, procedureInvalidated } from "./index-block.js";
+import { fmtDate, shortId } from "./util.js";
+
+/** Written into every generated SKILL.md; the ownership marker for safe cleanup. */
+export const SKILL_MARKER = "<!-- generated-by: dsh-engram skill-export -->";
+
+/** Where DSH's filesystem provider looks, highest precedence first. */
+export function defaultSkillDir(projectRoot) {
+  return path.join(projectRoot, ".dsh", "skills");
+}
+
+/** Fold arbitrary text into the host's kebab-case skill-name grammar. */
+export function kebab(text) {
+  return String(text ?? "")
+    .normalize("NFKD")
+    .replace(/[^\x00-\x7F]/g, "-")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-{2,}/g, "-");
+}
+
+/**
+ * `cmd:npm_test` → `npm-test`; the `cmd:` prefix is dropped because every
+ * exported skill is a command fix. Falls back to the memory id so the slug is
+ * never empty (an empty slug would produce an invalid skill name).
+ */
+export function skillSlug(row) {
+  const tag = (row?.tags ?? []).find((t) => typeof t === "string" && t.startsWith("cmd:"));
+  const base = kebab(tag !== undefined ? tag.slice(4) : row?.id ?? "");
+  const trimmed = base.slice(0, 48).replace(/-+$/, "");
+  return trimmed.length > 0 ? trimmed : `mem-${shortId(row?.id ?? "0")}`;
+}
+
+/** Host-legal skill name (regex from @deepseek-ai/dsh-skill). */
+export function skillName(row) {
+  return `engram-fix-${skillSlug(row)}`;
+}
+
+/** The command a procedure is about: its `cmd:` tag decoded, else the row text. */
+export function procedureCommand(row) {
+  const tag = (row?.tags ?? []).find((t) => typeof t === "string" && t.startsWith("cmd:"));
+  if (tag === undefined) return String(row?.text ?? "");
+  return tag.slice(4).replace(/_/g, " ");
+}
+
+/** One line for the host catalog: what it is, and how well it is attested. */
+export function skillDescription(row, now = Date.now()) {
+  const command = procedureCommand(row);
+  const count = row?.evidenceCount ?? 0;
+  const verified = row?.lastVerifiedAt !== null && row?.lastVerifiedAt !== undefined
+    ? fmtDate(row.lastVerifiedAt)
+    : "unknown";
+  return `Verified fix for \`${command}\` — held ${count}×, last verified ${verified}; engram procedure #${shortId(row?.id ?? "")}.`;
+}
+
+export function skillWhenToUse(row) {
+  const command = procedureCommand(row);
+  return `When \`${command}\` fails again and you want the fix that already held repeatedly on this machine.`;
+}
+
+/** Newest resolved error for the same command — the failure signature. */
+export function failureSignature(rows, row) {
+  const tag = (row?.tags ?? []).find((t) => typeof t === "string" && t.startsWith("cmd:"));
+  if (tag === undefined) return null;
+  const error = (rows ?? [])
+    .filter(
+      (m) =>
+        m?.kind === "error" &&
+        (m.tags ?? []).includes(tag) &&
+        (m.tags ?? []).includes("resolved") &&
+        m.id !== row.id,
+    )
+    .sort((a, b) => (b.createdAt ?? 0) - (a.createdAt ?? 0))[0];
+  if (error === undefined) return null;
+  return String(error.text ?? "").split("\n").slice(0, 3).join("\n").slice(0, 300);
+}
+
+/** Deterministic SKILL.md for one promoted procedure. */
+export function renderSkill(row, rows, { now = Date.now() } = {}) {
+  const command = procedureCommand(row);
+  const count = row?.evidenceCount ?? 0;
+  const first = row?.createdAt ?? null;
+  const last = row?.lastVerifiedAt ?? null;
+  const signature = failureSignature(rows, row);
+  const lines = [
+    "---",
+    `name: ${skillName(row)}`,
+    `description: ${JSON.stringify(skillDescription(row, now))}`,
+    `whenToUse: ${JSON.stringify(skillWhenToUse(row))}`,
+    "---",
+    "",
+    SKILL_MARKER,
+    `<!-- evidence: ${count} verified successes · last ${last !== null ? new Date(last).toISOString() : "unknown"} · memory ${row?.id ?? ""} -->`,
+    "",
+    `# Fix that holds: \`${command}\``,
+    "",
+    "## Evidence",
+    "",
+    `- verified successes: **${count}**`,
+    `- first captured: ${first !== null ? new Date(first).toISOString() : "unknown"}`,
+    `- last verified: ${last !== null ? new Date(last).toISOString() : "unknown"}`,
+    `- engram memory: \`${row?.id ?? ""}\``,
+    "",
+  ];
+  if (signature !== null) {
+    lines.push("## The failure it closes", "", "```", signature, "```", "");
+  }
+  lines.push(
+    "## How to use this",
+    "",
+    `1. Run \`${command}\` in this workspace first — the skill is evidence of a past`,
+    "   success, not a guarantee of the current state.",
+    "2. If it passes, the procedure still holds; nothing else to do.",
+    "3. If it fails, this skill is stale: dsh-engram invalidates the underlying",
+    "   procedure on that failure (its `[ENGRAM]` line disappears), so treat the",
+    "   fresh error as the truth and this note as history.",
+    "",
+    "## Provenance",
+    "",
+    "Generated by `dsh-engram skill export` from the memory store. Editing this",
+    "file is fine, but the next export of the same procedure rewrites it while the",
+    "marker above is present.",
+    "",
+  );
+  return lines.join("\n");
+}
+
+/**
+ * Pure planning step: which skills should exist, with what content. No I/O, so
+ * the rules are testable without touching a filesystem.
+ */
+export function planSkillExport(rows, { config, now = Date.now(), dir, approved = null, requireApproval = false }) {
+  const byName = new Map();
+  for (const row of rows ?? []) {
+    if (row === null || typeof row !== "object") continue;
+    if ((row.kind ?? "") !== "procedure") continue;
+    if (row.status === "archived") continue;
+    if (procedureInvalidated(row)) continue;
+    if (!isProvenProcedure(row, { ...config, nowMs: now })) continue;
+    if (!(row.tags ?? []).includes("auto-captured")) continue;
+    // v0.8-D: writing a file the host advertises needs a human yes, not only
+    // three successful runs. `approved` is the set of ids the user confirmed.
+    if (requireApproval && !(approved?.has?.(row.id) ?? false)) continue;
+    const name = skillName(row);
+    const previous = byName.get(name);
+    if (previous !== undefined && (previous.lastVerifiedAt ?? 0) >= (row.lastVerifiedAt ?? 0)) continue;
+    byName.set(name, row);
+  }
+  return [...byName.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([name, row]) => ({
+      name,
+      slug: skillSlug(row),
+      memoryId: row.id,
+      content: renderSkill(row, rows, { now }),
+      // Directory name is irrelevant to the host; using the skill name keeps
+      // the tree readable and the sweep comparison trivial.
+      // `dir` is optional: the GUI asks what WOULD be written without a target.
+      dir: typeof dir === "string" && dir.length > 0 ? path.join(dir, name) : null,
+    }));
+}
+
+/**
+ * Apply a plan to disk.
+ *
+ * - writes only when the bytes differ (repeated exports are no-ops, so the
+ *   host's watcher does not churn);
+ * - refuses to touch a SKILL.md without SKILL_MARKER (human-owned);
+ * - deletes generated SKILL.md files whose procedure no longer qualifies —
+ *   a broken procedure must not keep advertising itself as a skill.
+ */
+export function exportSkills(rows, { config, dir, now = Date.now(), dryRun = false, approved = null, requireApproval = false } = {}) {
+  const plan = planSkillExport(rows, { config, now, dir, approved, requireApproval });
+  const report = { written: [], unchanged: [], removed: [], skipped: [], pending: [], errors: [] };
+  // What an approval prompt would offer: proven, auto-captured, not yet approved.
+  if (requireApproval) {
+    for (const row of rows ?? []) {
+      if (row === null || typeof row !== "object") continue;
+      if ((row.kind ?? "") !== "procedure") continue;
+      if (row.status === "archived" || procedureInvalidated(row)) continue;
+      if (!isProvenProcedure(row, { ...config, nowMs: now })) continue;
+      if (!(row.tags ?? []).includes("auto-captured")) continue;
+      if (approved?.has?.(row.id) ?? false) continue;
+      report.pending.push({ id: row.id, name: skillName(row), description: skillDescription(row, now) });
+    }
+  }
+  const wanted = new Set(plan.map((skill) => skill.name));
+
+  if (!dryRun) {
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+    } catch (error) {
+      report.errors.push(`mkdir ${dir}: ${String(error)}`);
+      return report;
+    }
+  }
+
+  for (const skill of plan) {
+    const file = path.join(skill.dir, "SKILL.md");
+    let existing = null;
+    try {
+      existing = fs.readFileSync(file, "utf8");
+    } catch {
+      existing = null;
+    }
+    if (existing !== null && !existing.includes(SKILL_MARKER)) {
+      report.skipped.push(`${skill.name} (human-owned SKILL.md at ${file})`);
+      continue;
+    }
+    if (existing === skill.content) {
+      report.unchanged.push(skill.name);
+      continue;
+    }
+    if (!dryRun) {
+      try {
+        fs.mkdirSync(skill.dir, { recursive: true });
+        fs.writeFileSync(file, skill.content, "utf8");
+      } catch (error) {
+        report.errors.push(`${skill.name}: ${String(error)}`);
+        continue;
+      }
+    }
+    report.written.push(skill.name);
+  }
+
+  // Stale sweep: only our own marker files, only when their skill is not wanted
+  // any more (invalidated, expired, or the row was archived).
+  let entries = [];
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    entries = [];
+  }
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const file = path.join(dir, entry.name, "SKILL.md");
+    let content = null;
+    try {
+      content = fs.readFileSync(file, "utf8");
+    } catch {
+      continue;
+    }
+    if (!content.includes(SKILL_MARKER)) continue;
+    const name = /^name:\s*(.+)$/m.exec(content)?.[1]?.trim();
+    if (name !== undefined && wanted.has(name)) continue;
+    if (!dryRun) {
+      try {
+        fs.rmSync(file);
+        // Remove the directory too, but only when it holds nothing else.
+        if (fs.readdirSync(path.join(dir, entry.name)).length === 0) fs.rmdirSync(path.join(dir, entry.name));
+      } catch (error) {
+        report.errors.push(`remove ${entry.name}: ${String(error)}`);
+        continue;
+      }
+    }
+    report.removed.push(name ?? entry.name);
+  }
+
+  return report;
+}

package/lib/status-insights.js

@@ -0,0 +1,5 @@
+/**
+ * Shared predicates for CLI reporting (status/doctor). Thin re-exports so the
+ * CLI never imports the plugin entry (which pulls DSH packages).
+ */
+export { isProvenProcedure, procedureInvalidated as invalidatedProcedure } from "./index-block.js";

package/lib/status.js

@@ -25,6 +25,25 @@ export function contextGcStatusPath() {
   return join(engramHomeDir(), "engram", "context-gc.status.json");
 }
 
+/** How much a snapshot can say about the credit paths: 0 = nothing armed. */
+function rankHost(host) {
+  if (host === "context-gc") return 3;
+  if (host === "default") return 2;
+  if (host === "unavailable") return 1;
+  return 0;
+}
+
+/** Whether a pid is still running (EPERM counts as alive). */
+function isAlivePid(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
 /**
  * Atomically write the status snapshot (tmp + rename). Never throws for the
  * caller's convenience; a failure is logged by the caller if it cares.
@@ -33,6 +52,25 @@ export function contextGcStatusPath() {
  */
 export async function writeContextGcStatus(payload) {
   const path = contextGcStatusPath();
+  // Two plugin instances can settle in one process (the host row and a sibling
+  // realm that lost the `compaction` provider). Last-writer-wins then reports a
+  // working host as "unavailable" — a status file that lies is worse than none,
+  // so within one pid a *worse* snapshot never overwrites a better one.
+  const incoming = rankHost(payload?.host) + (payload?.boundaryPrune?.armed === true ? 1 : 0);
+  const existing = await readContextGcStatus().catch(() => void 0);
+  if (existing !== void 0 && existing.pid !== void 0) {
+    const current = rankHost(existing.host) + (existing?.boundaryPrune?.armed === true ? 1 : 0);
+    if (incoming < current) {
+      // Same pid: never downgrade (a sibling realm instance shares the process).
+      if (existing.pid === payload?.pid) return;
+      // Different pid: a LIVE writer keeps the file. Without this, any stray
+      // short-lived process that loads the plugin (a test run, a one-shot CLI,
+      // a headless probe with factory defaults) overwrites the running server's
+      // snapshot with "host unavailable / L1 not armed" — the status file lies,
+      // and every downstream reader believes it.
+      if (isAlivePid(existing.pid)) return;
+    }
+  }
   await mkdir(dirname(path), { recursive: true });
   const tmp = `${path}.tmp`;
   await writeFile(
@@ -55,3 +93,103 @@ export async function readContextGcStatus() {
     return undefined;
   }
 }
+
+/** Append-only log of compaction attempts (JSONL): the evidence that the
+ *  Context GC path actually ran — and the error when it did not. Without it a
+ *  broken engine looks exactly like a working one: the host silently falls
+ *  back to its default summarizer and the ledger credits nothing. */
+export function gcAttemptPath() {
+  return join(engramHomeDir(), "engram", "compaction-attempts.jsonl");
+}
+
+/**
+ * Record one attempt. Best effort: accounting/diagnostics must never break the
+ * compaction path. Keeps the tail bounded (rewrites to the last 20 lines past
+ * 32 KB) so a long-lived install cannot grow the file forever.
+ * @param {{ok: boolean, [key: string]: any}} payload
+ * @returns {Promise<void>}
+ */
+export async function recordGcAttempt(payload) {
+  try {
+    const file = gcAttemptPath();
+    await mkdir(dirname(file), { recursive: true });
+    const line = `${JSON.stringify({ at: new Date().toISOString(), ...payload })}\n`;
+    const { appendFile, readFile: read, stat, writeFile: write } = await import("node:fs/promises");
+    await appendFile(file, line, "utf8");
+    const info = await stat(file).catch(() => null);
+    if (info !== null && info.size > 32 * 1024) {
+      const text = await read(file, "utf8").catch(() => "");
+      const tail = text.trimEnd().split("\n").slice(-20).join("\n");
+      await write(file, `${tail}\n`, "utf8");
+    }
+  } catch {
+    /* diagnostics must never throw */
+  }
+}
+
+/** The most recent attempts (newest last), for the CLI and gate audit. */
+export async function readGcAttempts(limit = 20) {
+  try {
+    const { readFile } = await import("node:fs/promises");
+    const text = await readFile(gcAttemptPath(), "utf8");
+    return text
+      .split("\n")
+      .filter((line) => line.trim().length > 0)
+      .slice(-limit)
+      .map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/** Append-only log of boundary-prune attempts (JSONL). Same purpose as the
+ *  compaction attempts: L1 armed-but-never-firing must be distinguishable from
+ *  "no boundary ever happened". Every boundary the listener sees is recorded —
+ *  including ones the repayment/adaptive gate holds — with the plane it ran on
+ *  (host vs agent-preset realm), the pruned count and the chars removed. */
+export function pruneAttemptPath() {
+  return join(engramHomeDir(), "engram", "boundary-attempts.jsonl");
+}
+
+/** @param {{plane?: string, boundary?: any, pruned?: number, charsRemoved?: number, [key: string]: any}} payload */
+export async function recordPruneAttempt(payload) {
+  try {
+    const file = pruneAttemptPath();
+    await mkdir(dirname(file), { recursive: true });
+    const line = `${JSON.stringify({ at: new Date().toISOString(), ...payload })}\n`;
+    const { appendFile, readFile: read, stat, writeFile: write } = await import("node:fs/promises");
+    await appendFile(file, line, "utf8");
+    const info = await stat(file).catch(() => null);
+    if (info !== null && info.size > 32 * 1024) {
+      const text = await read(file, "utf8").catch(() => "");
+      const tail = text.trimEnd().split("\n").slice(-20).join("\n");
+      await write(file, `${tail}\n`, "utf8");
+    }
+  } catch {
+    /* diagnostics must never throw */
+  }
+}
+
+/** The most recent prune attempts (oldest first), for support and the CLI. */
+export async function readPruneAttempts(limit = 50) {
+  try {
+    const text = await readFile(pruneAttemptPath(), "utf8");
+    const lines = text.trimEnd().split("\n").filter(Boolean).slice(-limit);
+    return lines.map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    }).filter(Boolean);
+  } catch {
+    return [];
+  }
+}

package/lib/store.js

@@ -90,6 +90,15 @@ const MemorySchema = z.object({
   source: z.string().nullable().default(null),
   /** Optional trigger/condition under which this memory should be recalled. */
   conditions: z.string().nullable().default(null),
+  /** v0.8-A: procedure evidence. Count of verified successes since promotion
+   *  (1 at auto-capture time — one success is never enough to generalize). */
+  evidenceCount: z.number().int().nonnegative().default(0),
+  /** Last verified-success time (epoch ms). */
+  lastVerifiedAt: z.number().int().nullable().default(null),
+  /** Last time the same command failed again (epoch ms). */
+  lastFailedAt: z.number().int().nullable().default(null),
+  /** Memory id of the failure that invalidated this procedure. */
+  invalidatedBy: z.string().nullable().default(null),
 });
 
 const TaskSchema = z.object({
@@ -556,6 +565,10 @@ export async function openEngramDomain(facility) {
         contradicts,
         source: entry.source ?? null,
         conditions: entry.conditions ?? null,
+        evidenceCount: entry.evidenceCount ?? 0,
+        lastVerifiedAt: entry.lastVerifiedAt ?? null,
+        lastFailedAt: entry.lastFailedAt ?? null,
+        invalidatedBy: entry.invalidatedBy ?? null,
         expiresAt,
       };
       await memories.put(stored.id, stored);
@@ -688,6 +701,27 @@ export async function openEngramDomain(facility) {
       await memories.put(id, { ...m, tags });
       return true;
     },
+    /**
+     * v0.8-A procedure evidence accounting (zero LLM, non-appending).
+     *
+     * A `fixed:` procedure is only a claim until it keeps holding. Every later
+     * run of the same command either credits it (a verified success) or
+     * invalidates it (the command failed again) — the second is immediate and
+     * demotes the row out of the index, because a procedure that just broke is
+     * worse than no procedure: the next agent would copy it verbatim.
+     *
+     * Never touches `updatedAt`/`hits`: evidence is not recall, and crediting a
+     * success must not re-rank the row in recency order.
+     */
+    async recordProcedureOutcome(workspace, id, { ok, at = now(), errorId = null } = {}) {
+      const m = memories.get(id);
+      if (m === void 0 || m.workspace !== workspace || m.kind !== "procedure" || m.status !== "active") return null;
+      const next = ok
+        ? { ...m, evidenceCount: (m.evidenceCount ?? 0) + 1, lastVerifiedAt: at }
+        : { ...m, lastFailedAt: at, invalidatedBy: errorId ?? m.invalidatedBy ?? null, signal: Math.min(m.signal ?? 0.5, 0.2) };
+      await memories.put(id, next);
+      return next;
+    },
     /** Lazy TTL sweep (write path only; async). Returns archived count. */
     async expireSweep(workspace) {
       let n = 0;

package/lib/usage.js

@@ -265,6 +265,35 @@ function blockText(content) {
   return out;
 }
 
+/**
+ * Normalize one logged `tool/result` event across session formats:
+ *
+ *   v1–v3  data.message.content = [{ type: "tool-result", toolCallId, isError, content: [{ text }] }]
+ *   v4     data.message = { toolCallId, isError: true?, content: [{ type: "text", text }] }
+ *
+ * DSH 0.2.x writes v4. Reading only the wrapped shape (as `/stats` did until
+ * v0.7.5) makes every failure count and every recall-hits tally a silent zero —
+ * the same class of bug as `session.events` disappearing. Returns null when the
+ * event is not a tool result.
+ */
+export function readToolResult(event) {
+  const message = event?.data?.message;
+  if (message === null || typeof message !== "object") return null;
+  const blocks = Array.isArray(message.content) ? message.content : [];
+  const wrapped = blocks.find((b) => b?.type === "tool-result");
+  if (wrapped !== void 0) {
+    return {
+      callId: wrapped.toolCallId ?? message.toolCallId,
+      text: blockText(wrapped.content),
+      isError: wrapped.isError === true,
+    };
+  }
+  if (typeof message.toolCallId === "string" || blocks.some((b) => typeof b?.text === "string")) {
+    return { callId: message.toolCallId, text: blockText(blocks), isError: message.isError === true };
+  }
+  return null;
+}
+
 /**
  * Count tool calls across recent session logs.
  * @param opts - { root, days, now } — root is injectable for tests.
@@ -399,9 +428,9 @@ export function collectUsageStats({ root = sessionsRoot(), days = 14, now = Date
           continue;
         }
         if (j?.type !== "tool/result") continue;
-        const block = j.data?.message?.content?.find((c) => c?.type === "tool-result");
-        if (block?.toolCallId) {
-          results.set(block.toolCallId, { isError: block.isError === true, text: blockText(block.content) });
+        const parsed = readToolResult(j);
+        if (parsed !== null && typeof parsed.callId === "string" && parsed.callId.length > 0) {
+          results.set(parsed.callId, { isError: parsed.isError, text: parsed.text });
         }
       }
       let windowLeft = -1; // stream index of last hit-recall