@skill-harness/cli 0.4.0 → 0.6.0

package/assets/report.grade.js

@@ -143,7 +143,9 @@ export function liftNoneBadge(col) {
   if (col.lift) {
     return { text: "lift not comparable", title: col.liftHeadline || "nothing in the red baseline could be compared" };
   }
-  if (col.mode === "green") {
+  // Green and force are both skill-delivered, so both can be missing a baseline.
+  // A red column is the baseline and gets no badge at all.
+  if (col.mode === "green" || col.mode === "force") {
     return { text: "no red baseline", title: "run the same scenarios with --mode red to get a baseline" };
   }
   return null;

package/assets/report.template.html

@@ -43,6 +43,8 @@
   .cell .lm { display: block; font-size: 10px; font-weight: 700; }
   .cell .lm.up { color: var(--pass); }
   .cell .lm.down { color: var(--fail); }
+  /* Amber, like the suspect badge: both mean "this verdict is worth less than it looks". */
+  .cell .lm.flip { color: #b45309; }
   aside { width: 0; transition: width .15s ease; overflow: hidden; border-left: 1px solid var(--line); background: var(--panel); }
   aside.open { width: 460px; }
   .panel { width: 460px; padding: 16px 18px; }
@@ -126,10 +128,10 @@ function render() {
   const grades = DATA.columns.map((col) => gradeColumn(col, DATA.shipBar, DATA.critical));
   const queue = [];
   DATA.columns.forEach((col) => {
-    // A suspect cell in a red/force column can't be re-judged (/rejudge 400s
-    // "only green runs can be re-judged") — don't queue it. The matrix below
-    // still shows suspect badges for every column regardless of mode.
-    if (col.mode && col.mode !== "green") return;
+    // A suspect cell in a red column can't be re-judged from here (/rejudge 400s
+    // for an unscored baseline) — don't queue it. The matrix below still shows
+    // suspect badges for every column regardless of mode.
+    if (col.mode && col.mode !== "green" && col.mode !== "force") return;
     for (const scn of DATA.scenarios) {
       const cell = col.cells[scn.id];
       if (cell && cell.suspect && !cell.override) queue.push({ col: col.index, id: scn.id, label: col.label, reason: cell.judge_reason });
@@ -150,8 +152,8 @@ function render() {
   DATA.columns.forEach((col, i) => {
     const g = grades[i];
     let gradeHtml;
-    if (col.mode && col.mode !== "green") {
-      // Only green runs are scored; a red/force column has no ship grade.
+    if (col.mode && col.mode !== "green" && col.mode !== "force") {
+      // Green and force are scored; a red baseline column has no ship grade.
       gradeHtml = `<span class='badge no'>not scored (${escapeHtml(col.mode)})</span>`;
       if (g.suspect > 0) gradeHtml += ` — ${g.suspect} suspect`;
     } else {
@@ -220,7 +222,15 @@ function render() {
           : lc === "regressed"
             ? `<span class='lm down' title='red baseline PASS → ${effective(cell)} with the skill'>↓ skill</span>`
             : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${reps}</td>`;
+      // Run-over-run marker. Only on cells that actually flipped, and deliberately
+      // NOT folded into the flakiness number beside it: `flaky 0.00` is a within-run
+      // measure, and the case this exists for is a cell that was unanimous in every
+      // run and still landed on a different side each time.
+      const st = cell.stability;
+      const stabMark = st
+        ? `<span class='lm flip' title='${escapeHtml(st.note)}'>⇄ ${st.flips}/${st.compared}</span>`
+        : "";
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${reps}</td>`;
     });
     html += "</tr>";
   }
@@ -244,9 +254,10 @@ async function openPanel(colIndex, scenarioId) {
   panel.innerHTML = `
     <button class="close" id="closeBtn">×</button>
     <h2>${scn.id} · ${escapeHtml(scn.title)}</h2>
-    <div class="meta">${escapeHtml(col.label)} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
+    <div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
     <div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
     ${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
+    ${cell.stability ? `<div class="reason" style="color:#b45309"><b>⇄ boundary cell:</b> ${escapeHtml(cell.stability.note)}</div>` : ""}
     <div class="toggle">
       <button data-v="PASS" class="PASS ${cell.override === 'PASS' ? 'active PASS' : ''}">PASS</button>
       <button data-v="FAIL" class="FAIL ${cell.override === 'FAIL' ? 'active FAIL' : ''}">FAIL</button>
@@ -358,7 +369,12 @@ function renderTrends(data) {
     const badge = last.ship ? "<span class='badge ship'>SHIP</span>" : "<span class='badge no'>NOT READY</span>";
     const trunc = m.truncated ? ` <span class='dim'>(last ${m.runs.length})</span>` : "";
     const skippedNote = m.skipped > 0 ? ` <span class='dim'>(${m.skipped} unreadable)</span>` : "";
-    html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
+    // The mode is part of the series identity, not decoration: a tag that moved
+    // from green to force delivery has TWO series here, and the same skill text
+    // scores differently under each — so an unlabelled pair of sparklines would
+    // read as one history that jumped.
+    const modeNote = m.mode ? ` <span class='dim'>${escapeHtml(m.mode)}</span>` : "";
+    html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)}${modeNote} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
     html += "<table class='tgrid'><thead><tr><th></th>";
     for (const run of m.runs) html += `<th title="${escapeHtml(run.label || run.timestamp)}">${escapeHtml((run.label || run.timestamp).slice(0, 8))}</th>`;
     html += "</tr></thead><tbody>";

package/dist/cli.d.ts

@@ -23,7 +23,15 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
 export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function cmdInit(args: Args): Promise<void>;
 export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export declare function cmdLint(args: Args): Promise<void>;
 /**
  * The help text, rendered per call rather than frozen at module load.

package/dist/cli.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkd
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
 import yaml from "js-yaml";
-import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
@@ -134,6 +134,7 @@ export async function cmdRun(args) {
     if (!(await adapter.available()))
         throw new Error(`harness \`${harnessName}\` is not on PATH`);
     const mode = flagStr(args, "mode", "green") || "green";
+    const canary = flagBool(args, "canary");
     const label = flagStr(args, "label") || null;
     const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
     const { reps, passThreshold } = parseRunTuning(args);
@@ -174,6 +175,7 @@ export async function cmdRun(args) {
                 reps,
                 passThreshold,
                 only,
+                canary,
                 onProgress: (m) => console.log(m),
             });
             summaries.push(summary);
@@ -181,7 +183,11 @@ export async function cmdRun(args) {
             // any earlier run — the tag dir (<harness>-<modelslug>) is the join key.
             const tag = basename(dirname(summary.runDir));
             const lift = collectLift(skill.dir).find((l) => l.tag === tag);
-            console.log("\n" + formatScorecard(summary, lift) + "\n");
+            // Stability is derived from history INCLUDING the run just written, and scoped to
+            // this tag + mode: another model's flips under this model's scorecard would be a
+            // worse error than not reporting them at all.
+            const stability = collectStability(skill.dir).filter((c) => c.tag === tag && c.mode === summary.results.mode);
+            console.log("\n" + formatScorecard(summary, lift, stability) + "\n");
         }
     }
     console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
@@ -300,6 +306,68 @@ export async function cmdRegate(args, adapterOverride) {
     // flipped reps, which it must not spend silently.
     console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
 }
+/**
+ * Run-over-run verdict stability, derived from committed results. Free and offline: it
+ * reads results.yaml files and computes — no model, no judge, no harness.
+ *
+ * Exits 0 whatever it finds. A boundary cell is not a defect in the skill or in the
+ * spec; it is a statement about how much one run of that cell is worth. Making it a
+ * gate would turn "this needs more reps" into "your build is broken".
+ */
+async function cmdStability(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0] ?? "all";
+    const windowRaw = flagStr(args, "window");
+    const window = windowRaw ? Number(windowRaw) : undefined;
+    if (windowRaw !== undefined && (!Number.isInteger(window) || window < 2)) {
+        throw new Error(`--window must be an integer >= 2 (got \`${windowRaw}\`) — one run has no run-over-run step`);
+    }
+    const showAll = flagBool(args, "all");
+    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    if (skills.length === 0)
+        throw new Error(`no skills with a spec under ${root}`);
+    let boundaries = 0;
+    for (const skill of skills) {
+        const all = collectStability(skill.dir, { window });
+        if (all.length === 0) {
+            console.log(`\n${skill.name}: no scored runs yet — stability needs at least two runs of the same skill × model × mode`);
+            continue;
+        }
+        // One block per model tag × delivery mode: green and force are different
+        // deliveries of the same text, so their histories are never one series.
+        const groups = new Map();
+        for (const s of all) {
+            const key = `${s.tag} · mode=${s.mode}`;
+            (groups.get(key) ?? groups.set(key, []).get(key)).push(s);
+        }
+        console.log(`\n── ${skill.name} ──`);
+        for (const [key, cells] of groups) {
+            const runs = Math.max(...cells.map((c) => c.points.length));
+            console.log(`  ${key}  (${runs} run(s) in the window)`);
+            const boundary = boundaryCells(cells);
+            boundaries += boundary.length;
+            for (const s of boundary) {
+                console.log(`    ⇄ ${s.critical ? "CRITICAL " : ""}${stabilityNote(s)}`);
+            }
+            if (showAll) {
+                for (const s of cells) {
+                    if (s.state !== "boundary")
+                        console.log(`    ${s.state === "stable" ? "=" : "?"} ${stabilityNote(s)}`);
+                }
+            }
+            else {
+                const stable = cells.filter((c) => c.state === "stable").length;
+                const unmeasured = cells.filter((c) => c.state === "unmeasured").length;
+                console.log(`    ${stable} held their verdict · ${unmeasured} with no comparable step (--all to list them)`);
+            }
+        }
+    }
+    console.log(`\n${boundaries} boundary cell(s). ${PATH_LEGEND}`);
+    if (boundaries > 0) {
+        console.log(`A boundary cell is worth re-running with more reps (--reps) before you trust one run of it;`);
+        console.log(`within-run flakiness cannot see this, because it only ever looks at one run.`);
+    }
+}
 async function cmdReview(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0];
@@ -433,7 +501,15 @@ export async function cmdSuggest(args, adapterOverride) {
         rmSync(cwd, { recursive: true, force: true });
     }
 }
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export async function cmdLint(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0] ?? "all";
@@ -464,18 +540,20 @@ export async function cmdLint(args) {
             f = [{ skill: dir, code: "lint-error", message: e instanceof Error ? e.message : String(e) }];
         }
         findings.push(...f);
-        if (f.length === 0)
+        if (f.filter(failsGate).length === 0)
             console.log(`✓ ${dir}`);
-        else
-            for (const x of f) {
-                const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
-                console.log(`✗ ${where}: ${x.code} — ${x.message}`);
-                if (gha)
-                    console.log(`::error title=skill-harness::${where}: ${x.code} — ${x.message}`);
-            }
+        for (const x of f) {
+            const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
+            const fails = failsGate(x);
+            console.log(`${fails ? "✗" : "ℹ"} ${where}: ${x.code} — ${x.message}`);
+            if (gha)
+                console.log(`::${fails ? "error" : "notice"} title=skill-harness::${where}: ${x.code} — ${x.message}`);
+        }
     }
-    console.log(`\n${skillDirs.length} skill(s), ${findings.length} finding(s)`);
-    process.exitCode = findings.length > 0 ? 1 : 0;
+    const gating = findings.filter(failsGate).length;
+    const notes = findings.length - gating;
+    console.log(`\n${skillDirs.length} skill(s), ${gating} finding(s)${notes > 0 ? `, ${notes} note(s) (do not fail the gate)` : ""}`);
+    process.exitCode = gating > 0 ? 1 : 0;
 }
 // ---------------------------------------------------------------- dispatch
 /**
@@ -491,9 +569,11 @@ export function help() {
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
+                     [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
+  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (free, offline)
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
   init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
@@ -504,6 +584,9 @@ export function help() {
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
 defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=pi
+  green and force are both scored; red is the unscored baseline. green delivery depends on the
+  harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
+  so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
   the judge default is Opus on your Claude subscription (\`claude-code\` → \`claude -p\`), not a
   metered API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
 }
@@ -515,6 +598,7 @@ export async function main(argv) {
         case "grade": return cmdGrade(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
+        case "stability": return cmdStability(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
         case "init": return cmdInit(args);

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, envFlag, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, envFlag, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -99,9 +99,13 @@ export async function serveReview(opts) {
                     return;
                 }
                 const results = readResults(column.runDir);
-                if (results.mode !== "green") {
+                // Skill-delivered runs only (green or force). A red baseline's transcripts
+                // can be re-judged too — `skill-harness grade <run-dir>` does it — but a
+                // baseline has no grade for this endpoint to report back, and the button
+                // sits under a scorecard.
+                if (!isScoredMode(results.mode)) {
                     res.writeHead(400, { "content-type": "application/json" });
-                    res.end(JSON.stringify({ ok: false, error: "only green runs can be re-judged" }));
+                    res.end(JSON.stringify({ ok: false, error: `only scored runs (green/force) can be re-judged here — for a ${results.mode} run use \`skill-harness grade\`` }));
                     return;
                 }
                 const specPath = join(opts.skillDir, "tests", "specification.yaml");
@@ -126,13 +130,21 @@ export async function serveReview(opts) {
                 try {
                     const rr = await regradeScenario({
                         runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
-                        specDir: dirname(specPath), threshold,
+                        specDir: dirname(specPath), threshold, mode: results.mode,
                     });
                     const merged = results.scenarios.map((s) => s.id === body.scenarioId ? { ...rr, override: s.override, note: s.note } : s);
                     const written = writeResults(column.runDir, {
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
-                    }, { shipBar: spec.ship_bar, critical: spec.critical });
+                        partial: results.partial,
+                        // Provenance survives a UI re-judge, same as it does through `grade`.
+                        harness_cli_version: results.harness_cli_version, delivery_canary: results.delivery_canary,
+                        // Recorded hashes were being dropped here entirely, which silently
+                        // retired the staleness gate for any run re-judged from the UI. Carried,
+                        // with the one `rubric:` key this re-judge actually applied refreshed —
+                        // the same doctrine `grade` follows (see refreshRubricHashes).
+                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
+                    }, scoreContextFor(results, spec));
                     ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
                     const g = written.effective_grade;
                     appendJournal(column.runDir, { event: "score", ts: new Date().toISOString(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
@@ -167,11 +179,11 @@ export async function serveReview(opts) {
                     return;
                 }
                 // writeResults recomputes effective_grade override-aware against the CURRENT
-                // spec's ship bar — a saved override can never leave a stale grade. Only
-                // green runs are scored (PR #1 finding: /save must not grade red/force runs).
+                // spec's ship bar — a saved override can never leave a stale grade. Scored
+                // modes only: /save must not put a grade on a red baseline (PR #1 finding),
+                // and since 0.5.0 "scored" includes force.
                 const spec = loadSpec(join(opts.skillDir, "tests", "specification.yaml"));
-                const ctx = patched.mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
-                writeResults(column.runDir, patched, ctx);
+                writeResults(column.runDir, patched, scoreContextFor(patched, spec));
                 // Unconditional: a results root created before schema-2/journal.jsonl existed
                 // may still have a stale .gitignore body — every save (not just overrides)
                 // must roll it forward so journal.jsonl doesn't end up tracked.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.4.0",
+    "version": "0.6.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.4.0",
-        "@skill-harness/adapters": "0.4.0"
+        "@skill-harness/core": "0.6.0",
+        "@skill-harness/adapters": "0.6.0"
     }
 }