@skill-harness/cli 0.5.0 → 0.7.0

package/assets/report.template.html

@@ -43,6 +43,17 @@
   .cell .lm { display: block; font-size: 10px; font-weight: 700; }
   .cell .lm.up { color: var(--pass); }
   .cell .lm.down { color: var(--fail); }
+  /* Amber, like the suspect badge: both mean "this verdict is worth less than it looks". */
+  .cell .lm.flip { color: #b45309; }
+  /* Objective trace gates: pass/fail are facts about what the model did. ERROR is
+     amber rather than red — missing evidence is not a failing gate. */
+  .cell .lm.obj-pass { color: var(--pass); }
+  .cell .lm.obj-fail { color: var(--fail); }
+  .cell .lm.obj-error { color: #b45309; }
+  /* Adjudication. `unresolved` is amber for the same reason `suspect` is: it must
+     never read as a clean PASS or FAIL. */
+  .cell .lm.adj-settled { color: var(--dim); }
+  .cell .lm.adj-unresolved { color: #b45309; }
   aside { width: 0; transition: width .15s ease; overflow: hidden; border-left: 1px solid var(--line); background: var(--panel); }
   aside.open { width: 460px; }
   .panel { width: 460px; padding: 16px 18px; }
@@ -220,7 +231,27 @@ function render() {
           : lc === "regressed"
             ? `<span class='lm down' title='red baseline PASS → ${effective(cell)} with the skill'>↓ skill</span>`
             : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${reps}</td>`;
+      // Run-over-run marker. Only on cells that actually flipped, and deliberately
+      // NOT folded into the flakiness number beside it: `flaky 0.00` is a within-run
+      // measure, and the case this exists for is a cell that was unanimous in every
+      // run and still landed on a different side each time.
+      const st = cell.stability;
+      const stabMark = st
+        ? `<span class='lm flip' title='${escapeHtml(st.note)}'>⇄ ${st.flips}/${st.compared}</span>`
+        : "";
+      // Objective trace gates. Only rendered when the scenario DECLARED them —
+      // absent must read as "not declared", never as an objective pass.
+      const obj = cell.objective;
+      const objMark = obj
+        ? `<span class='lm obj-${obj.status.toLowerCase()}' title='${escapeHtml(`objective ${obj.status}: ${obj.detail}`)}'>◉ ${escapeHtml(obj.status.toLowerCase())}</span>`
+        : "";
+      // Adjudication. `unresolved` is amber and says so in words — a disagreement
+      // that renders as a clean verdict is the whole failure this feature prevents.
+      const adj = cell.adjudication;
+      const adjMark = adj
+        ? `<span class='lm adj-${adj.state === "unresolved" ? "unresolved" : "settled"}' title='${escapeHtml(`adjudication ${adj.state} (${adj.trigger}) · ${adj.detail}`)}'>⚖ ${adj.state === "unresolved" ? "unresolved" : `${adj.count} agree`}</span>`
+        : "";
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${objMark}${adjMark}${reps}</td>`;
     });
     html += "</tr>";
   }
@@ -247,12 +278,16 @@ async function openPanel(colIndex, scenarioId) {
     <div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
     <div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
     ${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
+    ${cell.stability ? `<div class="reason" style="color:#b45309"><b>⇄ boundary cell:</b> ${escapeHtml(cell.stability.note)}</div>` : ""}
+    ${cell.objective ? `<div class="reason"><b>◉ objective ${escapeHtml(cell.objective.status)}:</b> ${escapeHtml(cell.objective.detail)}</div>` : ""}
+    ${cell.adjudication ? `<div class="reason"${cell.adjudication.state === "unresolved" ? ' style="color:#b45309"' : ""}><b>⚖ adjudication ${escapeHtml(cell.adjudication.state)}</b> (${escapeHtml(cell.adjudication.trigger)}): ${escapeHtml(cell.adjudication.detail)}</div>` : ""}
     <div class="toggle">
       <button data-v="PASS" class="PASS ${cell.override === 'PASS' ? 'active PASS' : ''}">PASS</button>
       <button data-v="FAIL" class="FAIL ${cell.override === 'FAIL' ? 'active FAIL' : ''}">FAIL</button>
       <button data-v="" class="JUDGE ${!cell.override ? 'active JUDGE' : ''}">use judge (${cell.judge_verdict})</button>
     </div>
     <div class="rejudge"><button id="rejudgeBtn">Re-judge (${escapeHtml(col.judge.provider + ":" + col.judge.model)})</button> <span class="saved" id="rejudged"></span></div>
+    <div class="rejudge"><button id="adjBtn">Adjudicate column…</button> <span class="saved" id="adjudged"></span></div>
     <label class="fld">note <span class="saved" id="saved">saved ✓</span></label>
     <textarea id="note" placeholder="why you overrode / what to fix in SKILL.md">${escapeHtml(cell.note || "")}</textarea>
     <label class="fld">transcript</label>
@@ -277,6 +312,40 @@ async function openPanel(colIndex, scenarioId) {
       location.reload(); // re-judge rewrote results.yaml; reload the fresh matrix + grades
     } catch (e) { rj.disabled = false; rj.textContent = "Re-judge"; }
   };
+
+  // Two requests on purpose. The first prices the work and spends nothing; the
+  // confirm shows that exact ceiling; only then does anything call a judge. A
+  // single endpoint could not disclose a count before charging for it.
+  const adjBtn = document.getElementById("adjBtn");
+  if (adjBtn) adjBtn.onclick = async () => {
+    const status = document.getElementById("adjudged");
+    const show = (msg, err) => { if (status) { status.textContent = msg; status.classList.toggle("err", !!err); status.classList.add("show"); } };
+    adjBtn.disabled = true;
+    try {
+      const pr = await fetch("/adjudicate", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ col: colIndex, step: "plan" }) });
+      const plan = await pr.json().catch(() => ({}));
+      if (!pr.ok || !plan.ok) { show(plan.error || "could not plan adjudication", true); adjBtn.disabled = false; return; }
+      if (!plan.triggered.length) { show("no cell triggered — nothing to adjudicate", false); adjBtn.disabled = false; return; }
+
+      // Counts, never dollars: the default judge is a Claude subscription and
+      // reports no per-call usage, so a cost figure here would be invented.
+      const ok = window.confirm(
+        `Adjudicate ${plan.triggered.length} cell(s) with ${plan.judge}?\n\n` +
+        `Up to ${plan.maxAdditionalCalls} additional judge call(s).\n` +
+        `No tie-break judge from the browser — a disagreement stays unresolved and blocks SHIP.\n\n` +
+        plan.detail.join("\n")
+      );
+      if (!ok) { show("cancelled — nothing spent", false); adjBtn.disabled = false; return; }
+
+      adjBtn.textContent = "Adjudicating…";
+      const rr = await fetch("/adjudicate", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ col: colIndex, step: "run" }) });
+      const done = await rr.json().catch(() => ({}));
+      if (!rr.ok || !done.ok) { show(done.error || "adjudication failed", true); adjBtn.disabled = false; adjBtn.textContent = "Adjudicate column…"; return; }
+      location.reload();
+    } catch (e) {
+      show("adjudication failed", true); adjBtn.disabled = false; adjBtn.textContent = "Adjudicate column…";
+    }
+  };
   let t;
   document.getElementById("note").addEventListener("input", (e) => {
     cell.note = e.target.value;

package/dist/cli.d.ts

@@ -23,7 +23,15 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
 export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function cmdInit(args: Args): Promise<void>;
 export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export declare function cmdLint(args: Args): Promise<void>;
 /**
  * The help text, rendered per call rather than frozen at module load.

package/dist/cli.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
-import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
-import { basename, dirname, join, resolve } from "node:path";
+import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync, readdirSync } from "node:fs";
+import { load as yamlLoad } from "js-yaml";
+import { basename, dirname, join, resolve, relative } from "node:path";
 import { tmpdir } from "node:os";
-import yaml from "js-yaml";
-import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
@@ -129,6 +129,18 @@ export async function cmdRun(args) {
         source: judgeFlagRun ? "--judge" : "the default judge (SKILL_HARNESS_JUDGE or the baked value)",
         allowMetered: flagBool(args, "allow-metered-judge"),
     });
+    // Contradictory ARGUMENTS are refused before the environment is probed. Both
+    // checks reject the same command, but only one of them tells the user something
+    // they can act on: a machine without `pi` installed reported "harness `pi` is
+    // not on PATH" for a plain flag typo, sending the reader off to fix an install
+    // that was never the problem. Free, offline validation first — it is also why
+    // CI, which has no `pi`, could not test this refusal at all.
+    const onlyRaw = flagStr(args, "only");
+    let only = onlyRaw ? onlyRaw.split(",").map((x) => x.trim()).filter(Boolean) : undefined;
+    const affected = flagBool(args, "affected");
+    if (affected && only) {
+        throw new Error("--affected and --only both choose the scenario set — pass one, not both");
+    }
     const harnessName = flagStr(args, "harness", "pi");
     const adapter = getAdapter(harnessName);
     if (!(await adapter.available()))
@@ -138,8 +150,6 @@ export async function cmdRun(args) {
     const label = flagStr(args, "label") || null;
     const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
     const { reps, passThreshold } = parseRunTuning(args);
-    const onlyRaw = flagStr(args, "only");
-    const only = onlyRaw ? onlyRaw.split(",").map((x) => x.trim()).filter(Boolean) : undefined;
     const modelTokens = resolveModels(args);
     const skills = target === "all"
         ? discover(root).filter((s) => s.hasSpec)
@@ -154,6 +164,17 @@ export async function cmdRun(args) {
         // that look comparable and are not. Checked per skill, before its first token.
         assertNotDowngraded(skill.dir, "run");
         const spec = loadSpec(skill.specPath);
+        if (affected) {
+            // Reuses the exact `--only` machinery, so an affected run is partial and
+            // cannot report SHIP — the same guarantee, through the same code path.
+            const result = await computeAffected(args, spec.scenarios, skill.specPath);
+            console.log(formatAffected(result, spec.scenarios.length));
+            only = result.selected.map((sel) => sel.id);
+            if (only.length === 0) {
+                console.log(`skip ${skill.name}: no scenario is affected by this change`);
+                continue;
+            }
+        }
         for (const token of modelTokens) {
             const model = parseModelRef(token);
             // The version is on the banner because a stale global install is otherwise
@@ -183,7 +204,11 @@ export async function cmdRun(args) {
             // any earlier run — the tag dir (<harness>-<modelslug>) is the join key.
             const tag = basename(dirname(summary.runDir));
             const lift = collectLift(skill.dir).find((l) => l.tag === tag);
-            console.log("\n" + formatScorecard(summary, lift) + "\n");
+            // Stability is derived from history INCLUDING the run just written, and scoped to
+            // this tag + mode: another model's flips under this model's scorecard would be a
+            // worse error than not reporting them at all.
+            const stability = collectStability(skill.dir).filter((c) => c.tag === tag && c.mode === summary.results.mode);
+            console.log("\n" + formatScorecard(summary, lift, stability) + "\n");
         }
     }
     console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
@@ -227,7 +252,30 @@ export async function cmdGrade(args, adapterOverride) {
     for (const s of results.scenarios) {
         console.log(`  ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
     }
-    const g = results.effective_grade;
+    let final = results;
+    // Adjudication is opt-in. Without --auto-rejudge nothing below runs and not one
+    // extra call is made — a spec may declare triggers, but spec configuration alone
+    // never authorizes spending.
+    const judges = resolveAdjudicationJudges({
+        enabled: flagBool(args, "auto-rejudge"),
+        primary: judge,
+        secondaryToken: flagStr(args, "secondary-judge"),
+        tieBreakToken: flagStr(args, "tie-break-judge"),
+        subjectToken: results.model,
+        parseRef: parseModelRef,
+        assertAllowed: (j, source) => assertJudgeAllowed(j, { source, allowMetered: flagBool(args, "allow-metered-judge") }),
+        resemblesSubject: judgeResemblesSubject,
+        warn: (m) => console.error(m),
+    });
+    if (judges) {
+        final = await adjudicateRun({
+            runDir, spec, adapter, results, primaryJudge: judge,
+            secondaryJudge: judges.secondary, tieBreakJudge: judges.tieBreak,
+            specDir: testsDir, now: nowIso,
+            log: (m) => console.log(m),
+        });
+    }
+    const g = final.effective_grade;
     console.log(`\n  re-graded with ${judge.provider}:${judge.model} → ${g.letter} (${g.pct}%) ${g.ship ? "SHIP" : "NOT READY"}`);
 }
 /**
@@ -302,6 +350,68 @@ export async function cmdRegate(args, adapterOverride) {
     // flipped reps, which it must not spend silently.
     console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
 }
+/**
+ * Run-over-run verdict stability, derived from committed results. Free and offline: it
+ * reads results.yaml files and computes — no model, no judge, no harness.
+ *
+ * Exits 0 whatever it finds. A boundary cell is not a defect in the skill or in the
+ * spec; it is a statement about how much one run of that cell is worth. Making it a
+ * gate would turn "this needs more reps" into "your build is broken".
+ */
+async function cmdStability(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0] ?? "all";
+    const windowRaw = flagStr(args, "window");
+    const window = windowRaw ? Number(windowRaw) : undefined;
+    if (windowRaw !== undefined && (!Number.isInteger(window) || window < 2)) {
+        throw new Error(`--window must be an integer >= 2 (got \`${windowRaw}\`) — one run has no run-over-run step`);
+    }
+    const showAll = flagBool(args, "all");
+    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    if (skills.length === 0)
+        throw new Error(`no skills with a spec under ${root}`);
+    let boundaries = 0;
+    for (const skill of skills) {
+        const all = collectStability(skill.dir, { window });
+        if (all.length === 0) {
+            console.log(`\n${skill.name}: no scored runs yet — stability needs at least two runs of the same skill × model × mode`);
+            continue;
+        }
+        // One block per model tag × delivery mode: green and force are different
+        // deliveries of the same text, so their histories are never one series.
+        const groups = new Map();
+        for (const s of all) {
+            const key = `${s.tag} · mode=${s.mode}`;
+            (groups.get(key) ?? groups.set(key, []).get(key)).push(s);
+        }
+        console.log(`\n── ${skill.name} ──`);
+        for (const [key, cells] of groups) {
+            const runs = Math.max(...cells.map((c) => c.points.length));
+            console.log(`  ${key}  (${runs} run(s) in the window)`);
+            const boundary = boundaryCells(cells);
+            boundaries += boundary.length;
+            for (const s of boundary) {
+                console.log(`    ⇄ ${s.critical ? "CRITICAL " : ""}${stabilityNote(s)}`);
+            }
+            if (showAll) {
+                for (const s of cells) {
+                    if (s.state !== "boundary")
+                        console.log(`    ${s.state === "stable" ? "=" : "?"} ${stabilityNote(s)}`);
+                }
+            }
+            else {
+                const stable = cells.filter((c) => c.state === "stable").length;
+                const unmeasured = cells.filter((c) => c.state === "unmeasured").length;
+                console.log(`    ${stable} held their verdict · ${unmeasured} with no comparable step (--all to list them)`);
+            }
+        }
+    }
+    console.log(`\n${boundaries} boundary cell(s). ${PATH_LEGEND}`);
+    if (boundaries > 0) {
+        console.log(`A boundary cell is worth re-running with more reps (--reps) before you trust one run of it;`);
+        console.log(`within-run flakiness cannot see this, because it only ever looks at one run.`);
+    }
+}
 async function cmdReview(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0];
@@ -326,10 +436,6 @@ async function cmdAddTest(args) {
     if (!id || !title || turns.length === 0 || checks.length === 0) {
         throw new Error("add-test requires --id, --title, at least one --turn and one --check");
     }
-    // Validate the merged spec before writing.
-    const existing = loadSpec(skill.specPath);
-    if (existing.scenarios.some((s) => s.id === id))
-        throw new Error(`scenario id \`${id}\` already exists`);
     const scenario = { id, title };
     if (flagStr(args, "critical") !== undefined)
         scenario.critical = true;
@@ -340,12 +446,108 @@ async function cmdAddTest(args) {
     }
     scenario.turns = turns;
     scenario.checklist = checks;
-    const block = "\n" + yaml.dump({ scenarios: [scenario] }).replace(/^scenarios:\n/, "");
-    const merged = readFileSync(skill.specPath, "utf8") + block;
-    parseSpec(merged, skill.specPath); // throws if the append broke the spec
-    appendFileSync(skill.specPath, block, "utf8");
+    // Duplicate-id rejection, merged-spec validation and the atomic write all live
+    // in appendScenario — shared with capture promotion so the two paths cannot
+    // disagree about what a valid write is.
+    appendScenario({ specPath: skill.specPath, scenario });
     console.log(`added scenario ${id} to ${skill.specPath}`);
 }
+/**
+ * `coverage` — which instruction sections have a test declared against them.
+ *
+ * Free and offline. `--strict` turns uncovered sections into a non-zero exit, and
+ * is opt-in: an uncovered section is information, not a defect, and a linter that
+ * reddens CI for it teaches people to add a token `covers:` to silence it.
+ */
+async function cmdCoverage(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness coverage <skill|all> --skills <root> [--strict]");
+    const strict = flagBool(args, "strict");
+    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    let anyUncovered = false;
+    let anyBroken = false;
+    for (const skill of skills) {
+        if (!skill.hasSpec)
+            continue;
+        const spec = loadSpec(skill.specPath);
+        const specDir = dirname(skill.specPath);
+        const report = computeCoverage({
+            specDir,
+            scenarios: spec.scenarios,
+            // SKILL.md lives one level above tests/, and is the file `covers` almost
+            // always points at, so report on it even when nothing references it —
+            // otherwise a skill with zero `covers` reports 0 sections and looks fine.
+            baseFiles: [relative(specDir, join(skill.dir, "SKILL.md")).split("\\").join("/")],
+            pendingCaptures: readPendingCaptures(specDir),
+        });
+        console.log(formatCoverage(report, spec.skill));
+        if (report.uncovered.length)
+            anyUncovered = true;
+        if (report.broken.length)
+            anyBroken = true;
+    }
+    // A broken reference fails regardless of --strict: it is a wrong statement in
+    // the spec, not a gap in coverage.
+    if (anyBroken) {
+        console.error("\nbroken `covers` references above — fix the reference or the heading");
+        process.exitCode = 1;
+        return;
+    }
+    if (strict && anyUncovered) {
+        console.error("\n--strict: some sections have no declared test");
+        process.exitCode = 1;
+    }
+}
+/** Pending captures and the sections they are parked against. Free, offline, tolerant. */
+function readPendingCaptures(specDir) {
+    const dir = join(specDir, "captures");
+    if (!existsSync(dir))
+        return [];
+    const out = [];
+    for (const file of readdirSync(dir).filter((f) => f.endsWith(".yaml"))) {
+        try {
+            const raw = yamlLoad(readFileSync(join(dir, file), "utf8"));
+            if (!raw || raw.status === "promoted")
+                continue;
+            const covers = Array.isArray(raw.covers) ? raw.covers.filter((c) => typeof c === "string") : [];
+            if (covers.length)
+                out.push({ id: String(raw.id ?? file.replace(/\.yaml$/, "")), covers });
+        }
+        catch {
+            // A malformed capture is the capture command's problem to report; coverage
+            // must not fail because a draft file is mid-edit.
+        }
+    }
+    return out;
+}
+/** `affected` — which scenarios a change could plausibly touch. Free and offline. */
+async function cmdAffected(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness affected <skill> --skills <root> [--base <git-ref>]");
+    const skill = resolveSkill(root, target);
+    if (!skill.hasSpec)
+        throw new Error(`${target} has no spec`);
+    const spec = loadSpec(skill.specPath);
+    const result = await computeAffected(args, spec.scenarios, skill.specPath);
+    console.log(formatAffected(result, spec.scenarios.length));
+}
+/** Shared by `affected` and `run --affected`, so the two can never disagree. */
+async function computeAffected(args, scenarios, specPath) {
+    const base = flagStr(args, "base", "HEAD");
+    const repoRoot = await gitRepoRoot(dirname(specPath));
+    const diff = await gitDiff(repoRoot, base);
+    return selectAffected({ scenarios, specDir: dirname(specPath), diff, repoRoot });
+}
+async function gitRepoRoot(from) {
+    const r = await exec("git", ["rev-parse", "--show-toplevel"], { cwd: from, timeoutMs: 30_000 });
+    if (r.code !== 0)
+        throw new Error(`not a git repository (from ${from}) — --affected needs one to diff against`);
+    return r.stdout.trim();
+}
 /** Write a spec to disk, creating its tests/ dir. The single choke point for spec
  *  writes (init/suggest) so a future atomic-write/backup/audit change lands in one place. */
 function writeSpecFile(specPath, text) {
@@ -435,7 +637,15 @@ export async function cmdSuggest(args, adapterOverride) {
         rmSync(cwd, { recursive: true, force: true });
     }
 }
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export async function cmdLint(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0] ?? "all";
@@ -466,18 +676,20 @@ export async function cmdLint(args) {
             f = [{ skill: dir, code: "lint-error", message: e instanceof Error ? e.message : String(e) }];
         }
         findings.push(...f);
-        if (f.length === 0)
+        if (f.filter(failsGate).length === 0)
             console.log(`✓ ${dir}`);
-        else
-            for (const x of f) {
-                const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
-                console.log(`✗ ${where}: ${x.code} — ${x.message}`);
-                if (gha)
-                    console.log(`::error title=skill-harness::${where}: ${x.code} — ${x.message}`);
-            }
+        for (const x of f) {
+            const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
+            const fails = failsGate(x);
+            console.log(`${fails ? "✗" : "ℹ"} ${where}: ${x.code} — ${x.message}`);
+            if (gha)
+                console.log(`::${fails ? "error" : "notice"} title=skill-harness::${where}: ${x.code} — ${x.message}`);
+        }
     }
-    console.log(`\n${skillDirs.length} skill(s), ${findings.length} finding(s)`);
-    process.exitCode = findings.length > 0 ? 1 : 0;
+    const gating = findings.filter(failsGate).length;
+    const notes = findings.length - gating;
+    console.log(`\n${skillDirs.length} skill(s), ${gating} finding(s)${notes > 0 ? `, ${notes} note(s) (do not fail the gate)` : ""}`);
+    process.exitCode = gating > 0 ? 1 : 0;
 }
 // ---------------------------------------------------------------- dispatch
 /**
@@ -492,17 +704,24 @@ export function help() {
     return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
+                     [--affected --base <git-ref>]  run only the scenarios a change could touch (partial; never SHIPs)
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
                      [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
+                     [--auto-rejudge] [--secondary-judge p:m] [--tie-break-judge p:m]
+                       ask again about untrustworthy cells (ambiguous / contradictory / non-unanimous /
+                       ship-deciding). OFF by default; prints the exact MAX extra call count first.
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
+  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (free, offline)
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
   init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
   suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
   list   --skills <root>                        discovered skills + spec status
   lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
+  coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (free, offline)
+  affected <skill>   --skills <root> [--base ref]   which scenarios a change could touch (free, offline)
 
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
@@ -521,12 +740,15 @@ export async function main(argv) {
         case "grade": return cmdGrade(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
+        case "stability": return cmdStability(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
         case "init": return cmdInit(args);
         case "suggest": return cmdSuggest(args);
         case "list": return cmdList(args);
         case "lint": return cmdLint(args);
+        case "coverage": return cmdCoverage(args);
+        case "affected": return cmdAffected(args);
         case "version":
         case "--version":
         case "-v":

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, envFlag, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -132,7 +132,11 @@ export async function serveReview(opts) {
                         runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
                         specDir: dirname(specPath), threshold, mode: results.mode,
                     });
-                    const merged = results.scenarios.map((s) => s.id === body.scenarioId ? { ...rr, override: s.override, note: s.note } : s);
+                    const merged = results.scenarios.map((s) => 
+                    // Same contract as `grade`, through the same choke point.
+                    s.id === body.scenarioId
+                        ? rebuildScenarioResult(rr, s, { objective: "carry", adjudication: "drop" })
+                        : s);
                     const written = writeResults(column.runDir, {
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
@@ -160,6 +164,79 @@ export async function serveReview(opts) {
                 }
                 return;
             }
+            /**
+             * Adjudication, in two steps: `plan` returns the exact call ceiling and
+             * spends nothing; `run` executes it.
+             *
+             * Two requests rather than one because the ceiling has to be disclosed
+             * BEFORE anything is spent, and a single endpoint that both prices and
+             * charges cannot do that — the UI would be showing a count for work that
+             * had already happened.
+             */
+            if (req.method === "POST" && url.pathname === "/adjudicate") {
+                const body = JSON.parse((await readBody(req)) || "{}");
+                const data = collectReport(opts.skillDir);
+                const column = data.columns.find((c) => c.index === body.col);
+                if (!column) {
+                    res.writeHead(404).end("unknown column");
+                    return;
+                }
+                const results = readResults(column.runDir);
+                if (!isScoredMode(results.mode)) {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: `only scored runs (green/force) can be adjudicated — for a ${results.mode} run use \`skill-harness grade\`` }));
+                    return;
+                }
+                const specPath = join(opts.skillDir, "tests", "specification.yaml");
+                const spec = loadSpec(specPath);
+                const adapter = opts.adapter ?? getAdapter(results.harness);
+                // Priced exactly as `adjudicateRun` will perform it — including per-rep
+                // verdicts, without which the quoted ceiling can be lower than the spend.
+                const cells = cellsFromResults(column.runDir, results);
+                // No tie-break judge from the browser: adding a third judge is a judge
+                // CHOICE, and the UI has nowhere honest to make one. A disagreement here
+                // stays unresolved and blocks SHIP, which is the safe direction.
+                const plan = planAdjudication({
+                    cells, scenarios: spec.scenarios, shipBar: spec.ship_bar, critical: spec.critical,
+                    tieBreakAvailable: false,
+                });
+                if (body.step !== "run") {
+                    res.writeHead(200, { "content-type": "application/json" });
+                    res.end(JSON.stringify({
+                        ok: true, step: "plan",
+                        triggered: plan.triggered,
+                        maxAdditionalCalls: plan.maxAdditionalCalls,
+                        judge: `${results.judge.provider}:${results.judge.model}`,
+                        detail: plan.decisions.filter((d) => d.triggers.length).map((d) => `${d.id}: ${d.triggers.join(", ")}`),
+                    }));
+                    return;
+                }
+                if (!(await adapter.available())) {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: `harness \`${results.harness}\` is not on PATH` }));
+                    return;
+                }
+                try {
+                    assertJudgeAllowed(results.judge, { source: "the run's recorded judge", allowMetered: envFlag("SKILL_HARNESS_ALLOW_METERED_JUDGE") });
+                    const written = await adjudicateRun({
+                        runDir: column.runDir, spec, adapter, results,
+                        primaryJudge: results.judge,
+                        // Asked again as an independent draw. The judge-variance study measured
+                        // ~2% self-disagreement on identical transcripts, so this is a real
+                        // second opinion rather than a no-op.
+                        secondaryJudge: results.judge,
+                        specDir: dirname(specPath), now: () => new Date().toISOString(),
+                    });
+                    ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
+                    res.writeHead(200, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: true, step: "run", grade: written.effective_grade }));
+                }
+                catch (e) {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: e instanceof Error ? e.message : String(e) }));
+                }
+                return;
+            }
             if (req.method === "POST" && url.pathname === "/save") {
                 const body = JSON.parse((await readBody(req)) || "{}");
                 const data = collectReport(opts.skillDir);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.5.0",
+    "version": "0.7.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.5.0",
-        "@skill-harness/adapters": "0.5.0"
+        "@skill-harness/core": "0.7.0",
+        "@skill-harness/adapters": "0.7.0"
     }
 }