@skill-harness/cli 0.4.0 → 0.5.0

package/assets/report.grade.js

@@ -143,7 +143,9 @@ export function liftNoneBadge(col) {
   if (col.lift) {
     return { text: "lift not comparable", title: col.liftHeadline || "nothing in the red baseline could be compared" };
   }
-  if (col.mode === "green") {
+  // Green and force are both skill-delivered, so both can be missing a baseline.
+  // A red column is the baseline and gets no badge at all.
+  if (col.mode === "green" || col.mode === "force") {
     return { text: "no red baseline", title: "run the same scenarios with --mode red to get a baseline" };
   }
   return null;

package/assets/report.template.html

@@ -126,10 +126,10 @@ function render() {
   const grades = DATA.columns.map((col) => gradeColumn(col, DATA.shipBar, DATA.critical));
   const queue = [];
   DATA.columns.forEach((col) => {
-    // A suspect cell in a red/force column can't be re-judged (/rejudge 400s
-    // "only green runs can be re-judged") — don't queue it. The matrix below
-    // still shows suspect badges for every column regardless of mode.
-    if (col.mode && col.mode !== "green") return;
+    // A suspect cell in a red column can't be re-judged from here (/rejudge 400s
+    // for an unscored baseline) — don't queue it. The matrix below still shows
+    // suspect badges for every column regardless of mode.
+    if (col.mode && col.mode !== "green" && col.mode !== "force") return;
     for (const scn of DATA.scenarios) {
       const cell = col.cells[scn.id];
       if (cell && cell.suspect && !cell.override) queue.push({ col: col.index, id: scn.id, label: col.label, reason: cell.judge_reason });
@@ -150,8 +150,8 @@ function render() {
   DATA.columns.forEach((col, i) => {
     const g = grades[i];
     let gradeHtml;
-    if (col.mode && col.mode !== "green") {
-      // Only green runs are scored; a red/force column has no ship grade.
+    if (col.mode && col.mode !== "green" && col.mode !== "force") {
+      // Green and force are scored; a red baseline column has no ship grade.
       gradeHtml = `<span class='badge no'>not scored (${escapeHtml(col.mode)})</span>`;
       if (g.suspect > 0) gradeHtml += ` — ${g.suspect} suspect`;
     } else {
@@ -244,7 +244,7 @@ async function openPanel(colIndex, scenarioId) {
   panel.innerHTML = `
     <button class="close" id="closeBtn">×</button>
     <h2>${scn.id} · ${escapeHtml(scn.title)}</h2>
-    <div class="meta">${escapeHtml(col.label)} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
+    <div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
     <div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
     ${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
     <div class="toggle">
@@ -358,7 +358,12 @@ function renderTrends(data) {
     const badge = last.ship ? "<span class='badge ship'>SHIP</span>" : "<span class='badge no'>NOT READY</span>";
     const trunc = m.truncated ? ` <span class='dim'>(last ${m.runs.length})</span>` : "";
     const skippedNote = m.skipped > 0 ? ` <span class='dim'>(${m.skipped} unreadable)</span>` : "";
-    html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
+    // The mode is part of the series identity, not decoration: a tag that moved
+    // from green to force delivery has TWO series here, and the same skill text
+    // scores differently under each — so an unlabelled pair of sparklines would
+    // read as one history that jumped.
+    const modeNote = m.mode ? ` <span class='dim'>${escapeHtml(m.mode)}</span>` : "";
+    html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)}${modeNote} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
     html += "<table class='tgrid'><thead><tr><th></th>";
     for (const run of m.runs) html += `<th title="${escapeHtml(run.label || run.timestamp)}">${escapeHtml((run.label || run.timestamp).slice(0, 8))}</th>`;
     html += "</tr></thead><tbody>";

package/dist/cli.js

@@ -134,6 +134,7 @@ export async function cmdRun(args) {
     if (!(await adapter.available()))
         throw new Error(`harness \`${harnessName}\` is not on PATH`);
     const mode = flagStr(args, "mode", "green") || "green";
+    const canary = flagBool(args, "canary");
     const label = flagStr(args, "label") || null;
     const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
     const { reps, passThreshold } = parseRunTuning(args);
@@ -174,6 +175,7 @@ export async function cmdRun(args) {
                 reps,
                 passThreshold,
                 only,
+                canary,
                 onProgress: (m) => console.log(m),
             });
             summaries.push(summary);
@@ -491,6 +493,7 @@ export function help() {
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
+                     [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
@@ -504,6 +507,9 @@ export function help() {
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
 defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=pi
+  green and force are both scored; red is the unscored baseline. green delivery depends on the
+  harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
+  so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
   the judge default is Opus on your Claude subscription (\`claude-code\` → \`claude -p\`), not a
   metered API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
 }

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, envFlag, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, envFlag, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -99,9 +99,13 @@ export async function serveReview(opts) {
                     return;
                 }
                 const results = readResults(column.runDir);
-                if (results.mode !== "green") {
+                // Skill-delivered runs only (green or force). A red baseline's transcripts
+                // can be re-judged too — `skill-harness grade <run-dir>` does it — but a
+                // baseline has no grade for this endpoint to report back, and the button
+                // sits under a scorecard.
+                if (!isScoredMode(results.mode)) {
                     res.writeHead(400, { "content-type": "application/json" });
-                    res.end(JSON.stringify({ ok: false, error: "only green runs can be re-judged" }));
+                    res.end(JSON.stringify({ ok: false, error: `only scored runs (green/force) can be re-judged here — for a ${results.mode} run use \`skill-harness grade\`` }));
                     return;
                 }
                 const specPath = join(opts.skillDir, "tests", "specification.yaml");
@@ -126,13 +130,21 @@ export async function serveReview(opts) {
                 try {
                     const rr = await regradeScenario({
                         runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
-                        specDir: dirname(specPath), threshold,
+                        specDir: dirname(specPath), threshold, mode: results.mode,
                     });
                     const merged = results.scenarios.map((s) => s.id === body.scenarioId ? { ...rr, override: s.override, note: s.note } : s);
                     const written = writeResults(column.runDir, {
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
-                    }, { shipBar: spec.ship_bar, critical: spec.critical });
+                        partial: results.partial,
+                        // Provenance survives a UI re-judge, same as it does through `grade`.
+                        harness_cli_version: results.harness_cli_version, delivery_canary: results.delivery_canary,
+                        // Recorded hashes were being dropped here entirely, which silently
+                        // retired the staleness gate for any run re-judged from the UI. Carried,
+                        // with the one `rubric:` key this re-judge actually applied refreshed —
+                        // the same doctrine `grade` follows (see refreshRubricHashes).
+                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
+                    }, scoreContextFor(results, spec));
                     ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
                     const g = written.effective_grade;
                     appendJournal(column.runDir, { event: "score", ts: new Date().toISOString(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
@@ -167,11 +179,11 @@ export async function serveReview(opts) {
                     return;
                 }
                 // writeResults recomputes effective_grade override-aware against the CURRENT
-                // spec's ship bar — a saved override can never leave a stale grade. Only
-                // green runs are scored (PR #1 finding: /save must not grade red/force runs).
+                // spec's ship bar — a saved override can never leave a stale grade. Scored
+                // modes only: /save must not put a grade on a red baseline (PR #1 finding),
+                // and since 0.5.0 "scored" includes force.
                 const spec = loadSpec(join(opts.skillDir, "tests", "specification.yaml"));
-                const ctx = patched.mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
-                writeResults(column.runDir, patched, ctx);
+                writeResults(column.runDir, patched, scoreContextFor(patched, spec));
                 // Unconditional: a results root created before schema-2/journal.jsonl existed
                 // may still have a stale .gitignore body — every save (not just overrides)
                 // must roll it forward so journal.jsonl doesn't end up tracked.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.4.0",
+    "version": "0.5.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.4.0",
-        "@skill-harness/adapters": "0.4.0"
+        "@skill-harness/core": "0.5.0",
+        "@skill-harness/adapters": "0.5.0"
     }
 }