@supersuit/superskill 0.4.0 → 0.6.0

package/src/rules/superskill.mjs

@@ -1,14 +1,19 @@
-// Level 3, "superskill": checked against examples a person approved, fixed every time it
-// got something wrong, and proven recently on a current model against the no-skill baseline.
+// Level 3, "superskill" (0.6.0): fixed every time it got something wrong, with a regression eval
+// for each fix; its whole suite and trigger set pass against the SKILL.md that is here now, on a
+// current model and better than no skill; and it did the job for real people, one-shot, often
+// enough on at least one model+harness. Goldens are optional evidence.
 import { createHash } from "node:crypto";
 import { defineRules } from "./define.mjs";
-import { readGoldens, isApproved, weightOf } from "../goldens.mjs";
+import { readGoldens, isApproved, isRealApproved, weightOf, goldenOpts } from "../goldens.mjs";
+import { readRealRuns, meetsBar, knownPair, MIN_REAL_RUNS, MIN_ONE_SHOT } from "../realruns.mjs";
 import { readMisses } from "../misses.mjs";
 import { readEvals, readLatestRun } from "../evals.mjs";
 
 const f = (severity, message, fix) => ({ severity, message, fix });
 const DAY = 86400000;
-export const OPEN_MISS_DAYS = 14;
+/** Defaults for evals-pass; `--min-pass-rate` / `--min-trigger-rate` change them. */
+export const MIN_PASS_RATE = 0.9;
+export const MIN_TRIGGER_RATE = 0.9;
 /** How long a --run stays fresh, by metadata.cadence. */
 export const FRESH_DAYS = { daily: 30, weekly: 30, monthly: 60, quarterly: 120, yearly: 365 };
 const DEFAULT_FRESH = 30;
@@ -17,29 +22,56 @@ const days = (now, date) => Math.floor((now.getTime() - new Date(date).getTime()
 
 export const superskillRules = defineRules([
   {
+    // 0.6.0: a golden is optional evidence, never the bar. A random accepted run is not the
+    // embodiment of a skill; the misses it absorbed and the evals that hold them are. A golden
+    // that exists is still an eval (its checklist is graded by --run), so its files are checked.
     id: "golden-approved",
     level: "superskill",
-    check(ctx) {
+    check(ctx, opts = {}) {
       if (ctx.error) return [];
-      const goldens = readGoldens(ctx.dir);
-      const approved = goldens.filter(isApproved);
+      const goldens = readGoldens(ctx.dir, goldenOpts(ctx, opts));
+      if (!goldens.length) return [];
       const out = [];
+      const where = (g) => (g.private ? `private golden ${g.id}` : `goldens/${g.id}`);
       for (const g of goldens.filter((x) => x.approvalError))
-        out.push(f("fail", `goldens/${g.id}/APPROVAL.json is not valid JSON`, `Re-record it with \`superskill approve . ${g.id}\`.`));
-      if (!approved.length) {
-        out.push(f("fail", goldens.length ? `${goldens.length} golden${goldens.length === 1 ? "" : "s"}, none approved by a person` : "no goldens", goldens.length ? "A person runs `superskill approve <skill> <golden>` at a terminal after checking the output." : "Save a real input and the output you would sign off on under goldens/<id>/, then `superskill approve`."));
-        return out;
-      }
+        out.push(f("warn", `${where(g)}/APPROVAL.json is not valid JSON`, `Re-record it with \`superskill approve . ${g.id}\`.`));
+      for (const g of goldens.filter((x) => x.provenanceError))
+        out.push(f("warn", `${where(g)}/PROVENANCE.json is not valid JSON`, "Rewrite it: source, run, accepted (see SPEC.md, goldens)."));
+      for (const g of goldens.filter((x) => !x.expectations.length))
+        out.push(f("info", `${where(g)} has no expectations.json, so --run grades it by likeness to output.md`, "Write goldens/<id>/expectations.json: the behaviors a right answer shows (grade the outcome, not the path). output.md then stays as the reference that proves the task is solvable."));
+      const approved = goldens.filter(isApproved);
+      const real = goldens.filter(isRealApproved);
+      const unreal = approved.filter((g) => !real.includes(g));
+      if (unreal.length) out.push(f("info", `approved golden${unreal.length === 1 ? "" : "s"} not from a real run a person accepted: ${unreal.map((g) => `${g.id} ${g.origin.why}`).join("; ")}`, "Only a golden whose PROVENANCE.json names a real run and who accepted it is evidence of real use; an invented one is an ordinary eval."));
+      if (!real.length) return out;
       const sha = createHash("sha256").update(ctx.raw).digest("hex");
-      const stale = approved.filter((g) => g.approval.skill_sha && g.approval.skill_sha !== sha).map((g) => g.id);
+      const stale = real.filter((g) => g.approval.skill_sha && g.approval.skill_sha !== sha).map((g) => g.id);
       if (stale.length) out.push(f("info", `golden${stale.length === 1 ? "" : "s"} ${stale.join(", ")} approved against an earlier SKILL.md`, "Re-run the golden and re-approve if the output still holds."));
-      // Liked is not proven. Say which weight the approvals carry, so "a person approved it" is
-      // never read as "it worked in the world".
-      const w = approved.map((g) => ({ id: g.id, ...weightOf(g) }));
-      const proven = w.filter((x) => x.outcome > 0);
-      const summary = w.map((x) => `${x.id}: ${x.judgment} judgment, ${x.outcome} outcome`).join("; ");
-      if (!proven.length) out.push(f("info", `approved on judgment only, no outcome recorded yet (${summary})`, "When a golden produces a real result, record it: `superskill approve <skill> <golden> --basis outcome --evidence \"<what happened, where to check>\"`."));
-      else out.push(f("info", `approval weight: ${summary}`, ""));
+      const w = real.map((g) => ({ id: g.id, ...weightOf(g) }));
+      out.push(f("info", `real goldens (optional evidence): ${w.map((x) => `${x.id}: ${x.judgment} judgment, ${x.outcome} outcome`).join("; ")}`, ""));
+      return out;
+    },
+  },
+  {
+    // The record from real use, per model and harness. A clean record on one model in one harness
+    // proves nothing about another, so pairs are never pooled: one pair has to clear the bar alone.
+    id: "real-runs",
+    level: "superskill",
+    check(ctx, opts = {}) {
+      if (ctx.error) return [];
+      const name = (typeof ctx.data?.name === "string" && ctx.data.name) || ctx.folderName;
+      const minRuns = num(opts.minRealRuns, MIN_REAL_RUNS), minOneShot = num(opts.minOneShot, MIN_ONE_SHOT);
+      const bar = `at least ${minRuns} real runs and ${pct(minOneShot)} one-shot on one model+harness`;
+      const realRunsDir = opts.realRuns ?? ctx.realRuns ?? process.env.SUPERSKILL_REAL_RUNS ?? null;
+      const rec = readRealRuns(ctx.dir, { name, realRunsDir });
+      const how = "Export the run record (format superskill-real-runs/1, SPEC.md) to evals/real-runs.json or --real-runs <dir>; Freedom's skill ledger exports it.";
+      if (!rec) return [f("fail", `no real-run record (needs ${bar})`, how)];
+      if (rec.error) return [f("fail", rec.error, how)];
+      const out = rec.pairs.map((p) => f("info", `real runs, ${p.model} / ${p.harness}: ${p.one_shot} of ${p.runs} one-shot${p.runs ? ` (${pct(p.one_shot / p.runs)})` : ""}${knownPair(p) ? "" : ", not counted: the record does not say which model or harness"}`, ""));
+      if (!meetsBar(rec.pairs, { minRuns, minOneShot }).length) {
+        const best = rec.pairs.filter(knownPair)[0];
+        out.push(f("fail", `real-run record below the bar (${bar}; ${rec.where})${best ? `: best is ${best.model} / ${best.harness}, ${best.one_shot} of ${best.runs}` : ": no run names its model and harness"}`, "Use the skill for real and fix what it gets wrong; every correction is a miss to close with an eval."));
+      }
       return out;
     },
   },
@@ -52,18 +84,17 @@ export const superskillRules = defineRules([
     },
   },
   {
-    id: "no-stale-open-miss",
+    // 0.6.0: every miss is closed. An open miss is a known way the skill fails today, and a skill
+    // that still fails a known way is not at the top level, however recent the miss is.
+    id: "misses-closed",
     level: "superskill",
     check(ctx, { now = new Date() } = {}) {
       if (ctx.error) return [];
       const misses = readMisses(ctx.dir) || [];
-      const out = [];
-      for (const m of misses.filter((x) => x.status === "open")) {
+      return misses.filter((x) => x.status === "open").map((m) => {
         const age = days(now, m.date);
-        if (age > OPEN_MISS_DAYS) out.push(f("fail", `miss ${m.id} open ${age} days (limit ${OPEN_MISS_DAYS}): ${m.what.slice(0, 80)}`, `Fix it, add an eval that catches it, then \`superskill fix <skill> ${m.id} --eval <id>\`.`));
-        else out.push(f("info", `miss ${m.id} open ${age} day${age === 1 ? "" : "s"}`, `Fix within ${OPEN_MISS_DAYS} days of ${m.date}.`));
-      }
-      return out;
+        return f("fail", `miss ${m.id} is open (${age} day${age === 1 ? "" : "s"}): ${m.what.slice(0, 80)}`, `Fix it, add an eval that catches it, then \`superskill fix <skill> ${m.id} --eval <id>\`.`);
+      });
     },
   },
   {
@@ -74,12 +105,46 @@ export const superskillRules = defineRules([
       const misses = (readMisses(ctx.dir) || []).filter((m) => m.status === "fixed");
       if (!misses.length) return [];
       const ids = new Set(readEvals(ctx.dir).cases.map((c) => String(c.id)));
+      // Shipped goldens only: a miss is closed by a check that travels with the skill.
       for (const g of readGoldens(ctx.dir)) ids.add(g.id);
       return misses
         .filter((m) => !m.eval || !ids.has(String(m.eval)))
         .map((m) => f("fail", m.eval ? `miss ${m.id} names eval "${m.eval}", which is not in evals.json or goldens/` : `miss ${m.id} is fixed with no regression eval`, `Add a case to evals/evals.json that would catch ${m.id} again, and name its id on the Eval line.`));
     },
   },
+  {
+    // The suite and the trigger set pass against the SKILL.md that is here now. A pass recorded
+    // against an earlier SKILL.md proves the earlier skill.
+    id: "evals-pass",
+    level: "superskill",
+    check(ctx, opts = {}) {
+      if (ctx.error) return [];
+      const r = readLatestRun(ctx.dir);
+      if (!r.run) return [];
+      const run = r.run;
+      const out = [];
+      const sha = createHash("sha256").update(ctx.raw).digest("hex");
+      if (!run.skill_sha) out.push(f("fail", "latest.json does not say which SKILL.md it ran against (no skill_sha)", "Re-run `superskill doctor <skill> --run` with superskill 0.6.0 or later."));
+      else if (run.skill_sha !== sha) out.push(f("fail", "the last --run was against an earlier SKILL.md", "Re-run `superskill doctor <skill> --run`; a change to the skill needs its evals re-run."));
+      const minPass = num(opts.minPassRate, MIN_PASS_RATE), minTrig = num(opts.minTriggerRate, MIN_TRIGGER_RATE);
+      const w = Number(run.with_skill?.pass_rate);
+      if (Number.isFinite(w) && w < minPass) out.push(f("fail", `the suite passed ${pct(w)} of runs with the skill (needs ${pct(minPass)})`, "Read the failures in latest.json per_case, fix the skill, re-run."));
+      // Every fixed miss's regression eval passes every run: a regression that comes back half the
+      // time has come back.
+      const fixed = (readMisses(ctx.dir) || []).filter((m) => m.status === "fixed" && m.eval);
+      const cases = new Map((Array.isArray(run.per_case) ? run.per_case : []).map((c) => [String(c.id), c]));
+      for (const m of fixed) {
+        const c = cases.get(String(m.eval)) || cases.get(`golden:${m.eval}`);
+        if (!c) { if (run.skill_sha) out.push(f("fail", `regression eval ${m.eval} (miss ${m.id}) is not in the last --run`, "Re-run `superskill doctor <skill> --run`.")); continue; }
+        const ws = c.with_skill || {};
+        if (!(ws.runs > 0 && ws.passes === ws.runs)) out.push(f("fail", `regression eval ${m.eval} (miss ${m.id}) passed ${ws.passes || 0} of ${ws.runs || 0} runs`, `Miss ${m.id} is back: fix the skill until its eval passes every run.`));
+      }
+      const t = run.triggers;
+      if (!t || !Number.isFinite(Number(t.pass_rate))) out.push(f("fail", "the last --run did not run the trigger evals", "Re-run `superskill doctor <skill> --run` with superskill 0.6.0 or later; it runs evals/triggers.json too."));
+      else if (Number(t.pass_rate) < minTrig) out.push(f("fail", `trigger evals ${pct(Number(t.pass_rate))} right (needs ${pct(minTrig)})`, "Read triggers.per_query in latest.json: sharpen the description so it loads when it should and not otherwise."));
+      return out;
+    },
+  },
   {
     id: "run-evidence",
     level: "superskill",
@@ -118,3 +183,4 @@ export const superskillRules = defineRules([
 ]);
 
 const pct = (x) => `${Math.round(x * 100)}%`;
+const num = (v, d) => (v === undefined || v === null || v === "" || !Number.isFinite(Number(v)) ? d : Number(v));

package/src/rules/tested.mjs

@@ -2,7 +2,7 @@
 // with both should-load requests and near-misses that should not load the skill.
 import { defineRules } from "./define.mjs";
 import { readEvals, readTriggers } from "../evals.mjs";
-import { readGoldens } from "../goldens.mjs";
+import { readGoldens, goldenOpts } from "../goldens.mjs";
 
 const f = (severity, message, fix) => ({ severity, message, fix });
 export const MIN_EVALS = 3, MIN_TRIGGERS = 10, MIN_EACH_SIDE = 3;
@@ -15,7 +15,7 @@ export const testedRules = defineRules([
       if (ctx.error) return [];
       const e = readEvals(ctx.dir);
       if (e.error) return [f("fail", e.error, "Fix evals/evals.json so it parses as skill-creator's {skill_name, evals: [...]}.")];
-      const goldenCases = readGoldens(ctx.dir).filter((g) => g.input !== null && g.output !== null).length;
+      const goldenCases = readGoldens(ctx.dir, goldenOpts(ctx)).filter((g) => g.input !== null && g.output !== null).length;
       const n = e.cases.length + goldenCases;
       if (n < MIN_EVALS)
         return [f("fail", `${n} eval case${n === 1 ? "" : "s"} (need ${MIN_EVALS})`, "Add real requests to evals/evals.json (`superskill init` writes an example).")];
@@ -35,6 +35,29 @@ export const testedRules = defineRules([
         : [];
     },
   },
+  {
+    // MEASURED 2026-10-05 (freedom-dev): `superskill init` writes one example eval and two example
+    // triggers, each starting "REPLACE:", and a skill whose author copied those up to 3 and 10
+    // scored `tested` while testing nothing. A placeholder, or the same request twice, is not a case.
+    id: "evals-real",
+    level: "tested",
+    check(ctx) {
+      if (ctx.error) return [];
+      const e = readEvals(ctx.dir), t = readTriggers(ctx.dir);
+      const out = [];
+      const placeholder = (x) => /\bREPLACE:/.test(JSON.stringify(x));
+      const evalHits = e.cases.filter((c) => placeholder([c.prompt, c.expected_output, c.assertions])).map((c) => c.id);
+      if (evalHits.length) out.push(f("fail", `eval case${evalHits.length === 1 ? "" : "s"} ${evalHits.join(", ")} still hold${evalHits.length === 1 ? "s" : ""} a \`superskill init\` REPLACE: placeholder`, "Replace every example with a real request this skill handles and what a good answer must contain."));
+      const trigHits = t.triggers.filter((x) => placeholder(x.query)).length;
+      if (trigHits) out.push(f("fail", `${trigHits} trigger${trigHits === 1 ? "" : "s"} still hold a REPLACE: placeholder`, "Replace them with realistic requests, including near-misses that should not load the skill."));
+      const dup = (list) => list.filter((x, i) => x && list.indexOf(x) !== i);
+      const dp = [...new Set(dup(e.cases.map((c) => c.prompt.trim())))];
+      if (dp.length) out.push(f("fail", `${dp.length} eval prompt${dp.length === 1 ? " is" : "s are"} repeated: the same request twice is one case`, "Make each case a different request."));
+      const dq = [...new Set(dup(t.triggers.map((x) => x.query.trim().toLowerCase())))];
+      if (dq.length) out.push(f("fail", `${dq.length} trigger quer${dq.length === 1 ? "y is" : "ies are"} repeated`, "Make each trigger a different request."));
+      return out;
+    },
+  },
   {
     id: "triggers-present",
     level: "tested",

package/src/run/claude.mjs

@@ -3,12 +3,13 @@
 // a copy installed at user level cannot leak into the baseline.
 import { spawnSync } from "node:child_process";
 import { prepareWorkspace } from "./workspace.mjs";
+import { sandboxEnv } from "./env.mjs";
 
 export const name = "claude";
 
 function call(args, cwd) {
   const t0 = Date.now();
-  const r = spawnSync("claude", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  const r = spawnSync("claude", args, { cwd, env: sandboxEnv(), encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
   const ms = Date.now() - t0;
   if (r.error) throw Object.assign(new Error(`claude failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
   let doc = null;
@@ -34,3 +35,32 @@ export function ask(prompt, { model } = {}) {
   if (model) args.push("--model", model);
   return call(args, cwd).output;
 }
+
+/**
+ * Did Claude Code load the skill for this query? Run headless with the skill installed and read
+ * the stream: a Skill tool call naming it, or a Read of its SKILL.md, is a load.
+ */
+export function loadedSkill(stream, skillName) {
+  for (const line of String(stream).split("\n")) {
+    if (!line.trim().startsWith("{")) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const content = ev?.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const c of content) {
+      if (c?.type !== "tool_use") continue;
+      const input = c.input || {};
+      if (c.name === "Skill" && [input.skill, input.command, input.name].some((v) => typeof v === "string" && v.split(":").pop() === skillName)) return true;
+      if (typeof input.file_path === "string" && input.file_path.endsWith(`/${skillName}/SKILL.md`)) return true;
+    }
+  }
+  return false;
+}
+
+export function triggerCase({ skillDir, skillName, query, model }) {
+  const cwd = prepareWorkspace({ skillDir, skillName, linkAt: ".claude/skills" });
+  const args = ["-p", query, "--output-format", "stream-json", "--verbose", "--add-dir", cwd];
+  if (model) args.push("--model", model);
+  const r = spawnSync("claude", args, { cwd, env: sandboxEnv(), encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  if (r.error) throw Object.assign(new Error(`claude failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
+  return { triggered: loadedSkill(r.stdout, skillName), failed: r.status !== 0 && !r.stdout, cwd };
+}

package/src/run/codex.mjs

@@ -5,6 +5,7 @@ import { spawnSync } from "node:child_process";
 import { readFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { prepareWorkspace } from "./workspace.mjs";
+import { sandboxEnv } from "./env.mjs";
 
 export const name = "codex";
 
@@ -14,7 +15,7 @@ function call(prompt, cwd, model) {
   if (model) args.push("-m", model);
   args.push(prompt);
   const t0 = Date.now();
-  const r = spawnSync("codex", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  const r = spawnSync("codex", args, { cwd, env: sandboxEnv(), encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
   if (r.error) throw Object.assign(new Error(`codex failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
   const output = existsSync(out) ? readFileSync(out, "utf8") : r.stdout;
   const tok = (r.stderr + r.stdout).match(/tokens used[:\s]+([\d,]+)/i);
@@ -30,3 +31,17 @@ export function ask(prompt, { model } = {}) {
   const cwd = prepareWorkspace({ skillDir: null, skillName: null });
   return call(prompt, cwd, model).output;
 }
+
+/**
+ * Did Codex load the skill for this query? Codex reads a skill's SKILL.md when it uses it, so the
+ * JSON event stream mentioning <name>/SKILL.md is a load.
+ */
+export function triggerCase({ skillDir, skillName, query, model }) {
+  const cwd = prepareWorkspace({ skillDir, skillName, linkAt: ".agents/skills" });
+  const args = ["exec", "--json", "--skip-git-repo-check", "-C", cwd];
+  if (model) args.push("-m", model);
+  args.push(query);
+  const r = spawnSync("codex", args, { cwd, env: sandboxEnv(), encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  if (r.error) throw Object.assign(new Error(`codex failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
+  return { triggered: String(r.stdout).includes(`${skillName}/SKILL.md`), failed: r.status !== 0 && !r.stdout, cwd };
+}

package/src/run/env.mjs

@@ -0,0 +1,17 @@
+// The environment every sandboxed run gets, so a `doctor --run` session is never counted as a
+// real use of the skill by whatever is recording real uses.
+//
+// MEASURED, NOT FEARED (2026-10-05): Freedom's skill ledger records every skill run from a
+// harness hook. The doctor's own sandbox runs fired that hook, so three runs of a skill nobody had
+// ever used for real landed in the operator's ledger as perfect one-shot runs. Left alone, every
+// `doctor --run` raises the number that is meant to keep the doctor honest.
+//
+// Two signals, because a caller's recorder may honour either: FREEDOM_SKILL_LEDGER=off is the
+// Freedom ledger's documented off switch, and SUPERSKILL_SANDBOX=1 names the run for any other
+// recorder. The run folder's own name (SANDBOX_PREFIX) is the third, for a recorder that reads
+// only the hook payload's cwd.
+export const SANDBOX_PREFIX = "superskill-run-";
+
+export function sandboxEnv(base = process.env) {
+  return { ...base, FREEDOM_SKILL_LEDGER: "off", SUPERSKILL_SANDBOX: "1" };
+}

package/src/run/fake.mjs

@@ -1,14 +1,15 @@
 // A stand-in harness for tests: SUPERSKILL_FAKE_HARNESS names a node script that reads one
-// JSON request on stdin ({mode: "case"|"ask", prompt, withSkill, cwd}) and prints
-// {output, tokens?, model?}. Never calls a model.
+// JSON request on stdin ({mode: "case"|"ask"|"trigger", prompt|query, withSkill, cwd}) and
+// prints {output, tokens?, model?} (or {triggered} for a trigger). Never calls a model.
 import { spawnSync } from "node:child_process";
 import { prepareWorkspace } from "./workspace.mjs";
+import { sandboxEnv } from "./env.mjs";
 
 export const name = "fake";
 
 function call(script, req) {
   const t0 = Date.now();
-  const r = spawnSync(process.execPath, [script], { input: JSON.stringify(req), encoding: "utf8" });
+  const r = spawnSync(process.execPath, [script], { input: JSON.stringify(req), env: sandboxEnv(), encoding: "utf8" });
   if (r.status !== 0) throw Object.assign(new Error(`fake harness failed: ${r.stderr}`), { code: "SUPERSKILL" });
   const doc = JSON.parse(r.stdout);
   return { output: doc.output ?? "", tokens: doc.tokens ?? null, model: doc.model ?? "fake", ms: Date.now() - t0, failed: false };
@@ -21,6 +22,11 @@ export function create(script) {
       const cwd = prepareWorkspace({ skillDir, skillName, files, linkAt: withSkill ? ".claude/skills" : null });
       return { ...call(script, { mode: "case", prompt, withSkill, cwd }), cwd };
     },
+    triggerCase({ skillDir, skillName, query }) {
+      const cwd = prepareWorkspace({ skillDir, skillName, linkAt: ".claude/skills" });
+      const doc = JSON.parse(spawnSync(process.execPath, [script], { input: JSON.stringify({ mode: "trigger", query, cwd }), env: sandboxEnv(), encoding: "utf8" }).stdout || "{}");
+      return { triggered: Boolean(doc.triggered), failed: false, cwd };
+    },
     ask(prompt) {
       return call(script, { mode: "ask", prompt }).output;
     },

package/src/run/index.mjs

@@ -8,7 +8,8 @@ import { createInterface } from "node:readline/promises";
 import { clock, UsageError } from "../args.mjs";
 import { findSkills } from "../doctor.mjs";
 import { parseSkillFile } from "../frontmatter.mjs";
-import { readEvals } from "../evals.mjs";
+import { readEvals, readTriggers } from "../evals.mjs";
+import { createHash } from "node:crypto";
 import { readGoldens } from "../goldens.mjs";
 import { grade, isMachineCheck } from "./grade.mjs";
 import * as claude from "./claude.mjs";
@@ -18,7 +19,9 @@ import { create as createFake } from "./fake.mjs";
 export const help = `superskill doctor <skill> --run [--harness claude|codex] [--repeat 3] [--model <id>] [--yes]
 
 Run the skill's evals for real: every case in evals/evals.json and every golden, --repeat
-times with the skill and --repeat times without it, through a headless harness. Machine
+times with the skill and --repeat times without it, and every query in evals/triggers.json
+--repeat times with the skill installed (did it load when it should, and not otherwise),
+through a headless harness. Machine
 checks (contains:, regex:, file_exists:) are free; each plain-language expectation costs
 one grader call per run. Prints the estimated number of model calls first, and asks
 before starting (or needs --yes when there is no terminal). Writes
@@ -39,20 +42,57 @@ export function pickHarness(name) {
 }
 
 /** Every case the run will execute: evals.json cases plus goldens judged against their approved output. */
-export function collectCases(skillDir) {
+export function collectCases(skillDir, { privateGoldens = process.env.SUPERSKILL_PRIVATE_GOLDENS || null } = {}) {
   const e = readEvals(skillDir);
+  const name = parseSkillFile(readFileSync(join(skillDir, "SKILL.md"), "utf8")).data.name || "";
   if (e.error) throw new UsageError(e.error);
   const cases = e.cases.filter((c) => c.prompt.trim()).map((c) => ({ id: String(c.id), prompt: c.prompt, files: c.files, assertions: c.assertions.length ? c.assertions : [c.expected_output].filter(Boolean) }));
-  for (const g of readGoldens(skillDir)) {
-    if (!g.input || !g.output || !g.output.trim()) continue;
-    cases.push({ id: `golden:${g.id}`, prompt: g.input, files: [], assertions: [`The output matches this approved output in substance (same facts, same shape; wording may differ):\n${g.output}`] });
+  for (const g of readGoldens(skillDir, { privateGoldens, name })) {
+    if (!g.input || !((g.output && g.output.trim()) || g.expectations?.length)) continue;
+    // 0.6.0: grade the outcome, not the path. A golden with a checklist is graded on it; one
+    // without falls back to likeness with its reference output.
+    const assertions = g.expectations?.length ? g.expectations : [`The output matches this approved output in substance (same facts, same shape; wording may differ):\n${g.output}`];
+    cases.push({ id: `golden:${g.id}`, prompt: g.input, files: [], assertions });
   }
   return cases;
 }
 
-export function estimateCalls(cases, repeat) {
+export function estimateCalls(cases, repeat, triggers = 0) {
   const graded = cases.reduce((n, c) => n + c.assertions.filter((a) => !isMachineCheck(a)).length, 0);
-  return { runs: cases.length * repeat * 2, grader: graded * repeat * 2, total: cases.length * repeat * 2 + graded * repeat * 2 };
+  const runs = cases.length * repeat * 2 + triggers * repeat;
+  return { runs, grader: graded * repeat * 2, triggers: triggers * repeat, total: runs + graded * repeat * 2 };
+}
+
+/** The trigger queries the run will check, from evals/triggers.json. */
+export function collectTriggers(skillDir) {
+  const t = readTriggers(skillDir);
+  if (t.error) throw new UsageError(t.error);
+  return t.triggers;
+}
+
+/**
+ * Run every trigger query `repeat` times with the skill installed and record whether the harness
+ * loaded it. A query passes a run when loading matched should_trigger.
+ */
+export function runTriggers(skillDir, { harness, repeat, model, log = () => {} }) {
+  const skillName = parseSkillFile(readFileSync(join(skillDir, "SKILL.md"), "utf8")).data.name || "";
+  const queries = collectTriggers(skillDir);
+  if (!queries.length) return null;
+  if (typeof harness.triggerCase !== "function") throw new UsageError(`harness ${harness.name} cannot run trigger evals`);
+  let runs = 0, passes = 0;
+  const per_query = [];
+  for (const q of queries) {
+    const row = { query: q.query, should_trigger: q.should_trigger, runs: 0, passes: 0 };
+    for (let i = 0; i < repeat; i++) {
+      log(`trigger "${q.query.slice(0, 40)}", run ${i + 1}/${repeat}`);
+      const r = harness.triggerCase({ skillDir, skillName, query: q.query, model });
+      if (r.cwd) rmSync(r.cwd, { recursive: true, force: true });
+      const ok = !r.failed && Boolean(r.triggered) === q.should_trigger;
+      row.runs++; row.passes += ok ? 1 : 0; runs++; passes += ok ? 1 : 0;
+    }
+    per_query.push(row);
+  }
+  return { cases: queries.length, runs, passes, pass_rate: runs ? round(passes / runs) : 0, per_query };
 }
 
 export function runEvals(skillDir, { harness, repeat = 3, now = new Date(), model, log = () => {} }) {
@@ -83,7 +123,10 @@ export function runEvals(skillDir, { harness, repeat = 3, now = new Date(), mode
     per_case.push(row);
   }
   const summary = (t) => ({ pass_rate: t.runs ? round(t.passes / t.runs) : 0, mean_ms: t.runs ? Math.round(t.ms / t.runs) : 0, mean_tokens: t.tokenRuns ? Math.round(t.tokens / t.tokenRuns) : null });
-  const result = { run_at: now.toISOString(), harness: harness.name, model: seenModel, cases: cases.length, repeat, with_skill: summary(totals.with_skill), without_skill: summary(totals.without_skill), per_case };
+  const triggers = runTriggers(skillDir, { harness, repeat, model, log });
+  // 0.6.0: the run says which SKILL.md it proved, so a later edit cannot ride on an old pass.
+  const skill_sha = createHash("sha256").update(readFileSync(join(skillDir, "SKILL.md"), "utf8")).digest("hex");
+  const result = { run_at: now.toISOString(), harness: harness.name, model: seenModel, skill_sha, cases: cases.length, repeat, with_skill: summary(totals.with_skill), without_skill: summary(totals.without_skill), per_case, ...(triggers ? { triggers } : {}) };
   mkdirSync(join(skillDir, "evals", "results"), { recursive: true });
   writeFileSync(join(skillDir, "evals", "results", "latest.json"), JSON.stringify(result, null, 2) + "\n");
   return result;
@@ -99,11 +142,11 @@ export async function runCommand(a) {
   const harness = pickHarness(a.flags.harness);
   const dirs = a._.flatMap((p) => findSkills(p));
   if (!dirs.length) throw new UsageError(`no skills found under ${a._.join(", ")}`);
-  const plan = dirs.map((d) => ({ dir: d, cases: collectCases(d) }));
-  const calls = plan.reduce((n, p) => n + estimateCalls(p.cases, repeat).total, 0);
+  const plan = dirs.map((d) => ({ dir: d, cases: collectCases(d), triggers: collectTriggers(d).length }));
+  const calls = plan.reduce((n, p) => n + estimateCalls(p.cases, repeat, p.triggers).total, 0);
   for (const p of plan) {
-    const e = estimateCalls(p.cases, repeat);
-    process.stderr.write(`${p.dir}: ${p.cases.length} cases x ${repeat} x 2 = ${e.runs} runs + ${e.grader} grader calls\n`);
+    const e = estimateCalls(p.cases, repeat, p.triggers);
+    process.stderr.write(`${p.dir}: ${p.cases.length} cases x ${repeat} x 2 + ${p.triggers} triggers x ${repeat} = ${e.runs} runs + ${e.grader} grader calls\n`);
   }
   process.stderr.write(`estimated model calls: ${calls} through ${harness.name}\n`);
   if (!a.flags.yes) {
@@ -121,7 +164,7 @@ export async function runCommand(a) {
   for (const p of plan) {
     const r = runEvals(p.dir, { harness, repeat, now, model: a.flags.model, log: (m) => process.stderr.write(`  ${m}\n`) });
     results.push({ path: p.dir, ...r });
-    if (!a.flags.json) process.stdout.write(`${p.dir}\n  with skill ${pct(r.with_skill.pass_rate)}  without ${pct(r.without_skill.pass_rate)}  (${r.cases} cases x ${repeat})\n  wrote evals/results/latest.json\n`);
+    if (!a.flags.json) process.stdout.write(`${p.dir}\n  with skill ${pct(r.with_skill.pass_rate)}  without ${pct(r.without_skill.pass_rate)}  (${r.cases} cases x ${repeat})${r.triggers ? `\n  triggers ${pct(r.triggers.pass_rate)} right (${r.triggers.cases} queries x ${repeat})` : ""}\n  wrote evals/results/latest.json\n`);
   }
   if (a.flags.json) process.stdout.write(JSON.stringify({ results }, null, 2) + "\n");
   return results.every((r) => r.with_skill.pass_rate > r.without_skill.pass_rate) ? 0 : 1;

package/src/run/workspace.mjs

@@ -1,6 +1,7 @@
 import { mkdtempSync, mkdirSync, symlinkSync, cpSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
+import { SANDBOX_PREFIX } from "./env.mjs";
 
 /**
  * A fresh working folder for one run. With `linkAt` (e.g. ".claude/skills"), the skill is
@@ -8,7 +9,7 @@ import { join, dirname } from "node:path";
  * files (paths relative to the skill) are copied in at the same relative paths.
  */
 export function prepareWorkspace({ skillDir, skillName, files = [], linkAt = null }) {
-  const cwd = mkdtempSync(join(tmpdir(), "superskill-run-"));
+  const cwd = mkdtempSync(join(tmpdir(), SANDBOX_PREFIX));
   if (linkAt) {
     const target = join(cwd, linkAt, skillName);
     mkdirSync(dirname(target), { recursive: true });

package/src/snippet.md

@@ -5,12 +5,14 @@ Skills in this project are held to the superskill standard: https://supersuit.wi
 - **After doing a job by hand once, propose a skill for it.** If the person agrees, write the
   skill, then seed its evidence from this session:
   `npx @supersuit/superskill init <skill-folder> --from-session <this session's transcript>`
-  The golden it creates waits for the person to approve it; never approve it yourself.
+  The golden it creates is optional evidence and waits for the person; never approve it yourself.
 - **Whenever a skill needed correcting** (the person redirected you, fixed its output, or you
   worked around it), log it before moving on:
   `npx @supersuit/superskill miss <skill-folder> "<what happened>" --expected "<what should have>"`
   When you fix it, add an eval that would catch it again and close it with
   `npx @supersuit/superskill fix <skill-folder> <miss-id> --eval <eval-id>`.
+- **The story goes in MISSES.md, never in SKILL.md.** SKILL.md holds the rule and a one-line
+  why; what happened, when, and what the person said go on the miss entry (`--quote`).
 - **Before calling a skill done**, run
   `npx @supersuit/superskill doctor <skill-folder>`
   and fix everything it lists for the level the skill is meant to reach.