@skill-harness/cli 0.5.0 → 0.6.0

package/assets/report.template.html

@@ -43,6 +43,8 @@
   .cell .lm { display: block; font-size: 10px; font-weight: 700; }
   .cell .lm.up { color: var(--pass); }
   .cell .lm.down { color: var(--fail); }
+  /* Amber, like the suspect badge: both mean "this verdict is worth less than it looks". */
+  .cell .lm.flip { color: #b45309; }
   aside { width: 0; transition: width .15s ease; overflow: hidden; border-left: 1px solid var(--line); background: var(--panel); }
   aside.open { width: 460px; }
   .panel { width: 460px; padding: 16px 18px; }
@@ -220,7 +222,15 @@ function render() {
           : lc === "regressed"
             ? `<span class='lm down' title='red baseline PASS → ${effective(cell)} with the skill'>↓ skill</span>`
             : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${reps}</td>`;
+      // Run-over-run marker. Only on cells that actually flipped, and deliberately
+      // NOT folded into the flakiness number beside it: `flaky 0.00` is a within-run
+      // measure, and the case this exists for is a cell that was unanimous in every
+      // run and still landed on a different side each time.
+      const st = cell.stability;
+      const stabMark = st
+        ? `<span class='lm flip' title='${escapeHtml(st.note)}'>⇄ ${st.flips}/${st.compared}</span>`
+        : "";
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${reps}</td>`;
     });
     html += "</tr>";
   }
@@ -247,6 +257,7 @@ async function openPanel(colIndex, scenarioId) {
     <div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
     <div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
     ${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
+    ${cell.stability ? `<div class="reason" style="color:#b45309"><b>⇄ boundary cell:</b> ${escapeHtml(cell.stability.note)}</div>` : ""}
     <div class="toggle">
       <button data-v="PASS" class="PASS ${cell.override === 'PASS' ? 'active PASS' : ''}">PASS</button>
       <button data-v="FAIL" class="FAIL ${cell.override === 'FAIL' ? 'active FAIL' : ''}">FAIL</button>

package/dist/cli.d.ts

@@ -23,7 +23,15 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
 export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function cmdInit(args: Args): Promise<void>;
 export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export declare function cmdLint(args: Args): Promise<void>;
 /**
  * The help text, rendered per call rather than frozen at module load.

package/dist/cli.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkd
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
 import yaml from "js-yaml";
-import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
@@ -183,7 +183,11 @@ export async function cmdRun(args) {
             // any earlier run — the tag dir (<harness>-<modelslug>) is the join key.
             const tag = basename(dirname(summary.runDir));
             const lift = collectLift(skill.dir).find((l) => l.tag === tag);
-            console.log("\n" + formatScorecard(summary, lift) + "\n");
+            // Stability is derived from history INCLUDING the run just written, and scoped to
+            // this tag + mode: another model's flips under this model's scorecard would be a
+            // worse error than not reporting them at all.
+            const stability = collectStability(skill.dir).filter((c) => c.tag === tag && c.mode === summary.results.mode);
+            console.log("\n" + formatScorecard(summary, lift, stability) + "\n");
         }
     }
     console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
@@ -302,6 +306,68 @@ export async function cmdRegate(args, adapterOverride) {
     // flipped reps, which it must not spend silently.
     console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
 }
+/**
+ * Run-over-run verdict stability, derived from committed results. Free and offline: it
+ * reads results.yaml files and computes — no model, no judge, no harness.
+ *
+ * Exits 0 whatever it finds. A boundary cell is not a defect in the skill or in the
+ * spec; it is a statement about how much one run of that cell is worth. Making it a
+ * gate would turn "this needs more reps" into "your build is broken".
+ */
+async function cmdStability(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0] ?? "all";
+    const windowRaw = flagStr(args, "window");
+    const window = windowRaw ? Number(windowRaw) : undefined;
+    if (windowRaw !== undefined && (!Number.isInteger(window) || window < 2)) {
+        throw new Error(`--window must be an integer >= 2 (got \`${windowRaw}\`) — one run has no run-over-run step`);
+    }
+    const showAll = flagBool(args, "all");
+    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    if (skills.length === 0)
+        throw new Error(`no skills with a spec under ${root}`);
+    let boundaries = 0;
+    for (const skill of skills) {
+        const all = collectStability(skill.dir, { window });
+        if (all.length === 0) {
+            console.log(`\n${skill.name}: no scored runs yet — stability needs at least two runs of the same skill × model × mode`);
+            continue;
+        }
+        // One block per model tag × delivery mode: green and force are different
+        // deliveries of the same text, so their histories are never one series.
+        const groups = new Map();
+        for (const s of all) {
+            const key = `${s.tag} · mode=${s.mode}`;
+            (groups.get(key) ?? groups.set(key, []).get(key)).push(s);
+        }
+        console.log(`\n── ${skill.name} ──`);
+        for (const [key, cells] of groups) {
+            const runs = Math.max(...cells.map((c) => c.points.length));
+            console.log(`  ${key}  (${runs} run(s) in the window)`);
+            const boundary = boundaryCells(cells);
+            boundaries += boundary.length;
+            for (const s of boundary) {
+                console.log(`    ⇄ ${s.critical ? "CRITICAL " : ""}${stabilityNote(s)}`);
+            }
+            if (showAll) {
+                for (const s of cells) {
+                    if (s.state !== "boundary")
+                        console.log(`    ${s.state === "stable" ? "=" : "?"} ${stabilityNote(s)}`);
+                }
+            }
+            else {
+                const stable = cells.filter((c) => c.state === "stable").length;
+                const unmeasured = cells.filter((c) => c.state === "unmeasured").length;
+                console.log(`    ${stable} held their verdict · ${unmeasured} with no comparable step (--all to list them)`);
+            }
+        }
+    }
+    console.log(`\n${boundaries} boundary cell(s). ${PATH_LEGEND}`);
+    if (boundaries > 0) {
+        console.log(`A boundary cell is worth re-running with more reps (--reps) before you trust one run of it;`);
+        console.log(`within-run flakiness cannot see this, because it only ever looks at one run.`);
+    }
+}
 async function cmdReview(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0];
@@ -435,7 +501,15 @@ export async function cmdSuggest(args, adapterOverride) {
         rmSync(cwd, { recursive: true, force: true });
     }
 }
-/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
+/**
+ * Exit-code contract: 0 = no gate-failing findings, 1 = >=1 of them, or a resolution
+ * error (unknown skill/root, no skills with a spec).
+ *
+ * `info` findings (run-over-run stability notes) print and annotate but never fail the
+ * gate: a boundary cell says how much one run of a scenario is worth, which is not a
+ * defect in the spec, the fixtures or the results. A linter that reddens CI for it would
+ * teach everyone to stop reading it.
+ */
 export async function cmdLint(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0] ?? "all";
@@ -466,18 +540,20 @@ export async function cmdLint(args) {
             f = [{ skill: dir, code: "lint-error", message: e instanceof Error ? e.message : String(e) }];
         }
         findings.push(...f);
-        if (f.length === 0)
+        if (f.filter(failsGate).length === 0)
             console.log(`✓ ${dir}`);
-        else
-            for (const x of f) {
-                const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
-                console.log(`✗ ${where}: ${x.code} — ${x.message}`);
-                if (gha)
-                    console.log(`::error title=skill-harness::${where}: ${x.code} — ${x.message}`);
-            }
+        for (const x of f) {
+            const where = x.scenario ? `${dir}/${x.scenario}` : dir; // dir-based label, consistent with the ✓ line
+            const fails = failsGate(x);
+            console.log(`${fails ? "✗" : "ℹ"} ${where}: ${x.code} — ${x.message}`);
+            if (gha)
+                console.log(`::${fails ? "error" : "notice"} title=skill-harness::${where}: ${x.code} — ${x.message}`);
+        }
     }
-    console.log(`\n${skillDirs.length} skill(s), ${findings.length} finding(s)`);
-    process.exitCode = findings.length > 0 ? 1 : 0;
+    const gating = findings.filter(failsGate).length;
+    const notes = findings.length - gating;
+    console.log(`\n${skillDirs.length} skill(s), ${gating} finding(s)${notes > 0 ? `, ${notes} note(s) (do not fail the gate)` : ""}`);
+    process.exitCode = gating > 0 ? 1 : 0;
 }
 // ---------------------------------------------------------------- dispatch
 /**
@@ -497,6 +573,7 @@ export function help() {
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
+  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (free, offline)
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
   init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
@@ -521,6 +598,7 @@ export async function main(argv) {
         case "grade": return cmdGrade(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
+        case "stability": return cmdStability(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
         case "init": return cmdInit(args);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.5.0",
+    "version": "0.6.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.5.0",
-        "@skill-harness/adapters": "0.5.0"
+        "@skill-harness/core": "0.6.0",
+        "@skill-harness/adapters": "0.6.0"
     }
 }