@skill-harness/cli 0.22.0 → 0.22.4

package/assets/report.grade.js

@@ -55,6 +55,7 @@ export function gradeColumn(col, shipBar, critical) {
   let suspect = 0;
   let errors = 0;
   let notMeasured = 0;
+  let ungraded = 0;
 
   for (const id of Object.keys(col.cells)) {
     const cell = col.cells[id];
@@ -77,12 +78,13 @@ export function gradeColumn(col, shipBar, critical) {
       passed++;
       continue;
     }
+    if (verdict === "UNGRADED") ungraded++;
     if (critical.includes(id)) criticalFails++;
     if (/^B/i.test(id)) bFails++;
   }
 
   if (col.partial === true) {
-    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured };
+    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured, ungraded };
   }
 
   const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
@@ -98,7 +100,7 @@ export function gradeColumn(col, shipBar, critical) {
     errors === 0 &&
     notMeasured === 0;
 
-  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured };
+  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured, ungraded };
 }
 
 /**
@@ -114,7 +116,7 @@ export function gradeColumn(col, shipBar, critical) {
  */
 export function liftClass(liftCell, greenCell) {
   const conclusive = (verdict, suspect) =>
-    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
+    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS" && verdict !== "UNGRADED";
 
   const redOk = conclusive(liftCell.red, liftCell.redSuspect);
   const greenOk = conclusive(effective(greenCell), !!greenCell.suspect && !greenCell.override);

package/assets/report.template.html

@@ -25,6 +25,7 @@
   .cell.PASS { background: rgba(31,139,76,.22); }
   .cell.FAIL { background: rgba(192,57,43,.24); }
   .cell.ERROR { background: rgba(107,114,128,.22); }
+  .cell.UNGRADED { background: rgba(180,83,9,.18); }
   .cell.NOT-MEASURED { background: rgba(180,83,9,.18); }
   .cell.empty { color: var(--dim); cursor: default; font-weight: 400; }
   .cell .ov { display: block; font-size: 11px; color: var(--over); font-weight: 600; }
@@ -229,6 +230,7 @@ function render() {
         ? `<span class='ov unsaved'>unsaved</span>`
         : (cell.override ? `<span class='ov'>override</span>` : "");
       const misfired = cell.clean != null && cell.clean < cell.reps ? ` · ${cell.reps - cell.clean} misfired` : "";
+      const ungraded = cell.ungraded_reps ? `<span class='reps'>${cell.ungraded_reps} UNGRADED rep${cell.ungraded_reps === 1 ? "" : "s"} (not judged FAIL)</span>` : "";
       const reps = cell.reps ? `<span class='reps'>${cell.passes}/${cell.clean}${misfired}${cell.flakiness ? ` · flaky ${cell.flakiness.toFixed(2)}` : ""}</span>` : "";
       const suspectBadge = cell.suspect && !cell.override ? `<span class='ov suspect'>suspect</span>` : "";
       // Per-cell lift marker: only the two classes that mean the skill changed
@@ -255,7 +257,7 @@ function render() {
       const objMark = obj
         ? `<span class='lm obj-${obj.status.toLowerCase()}' title='${escapeHtml(`objective ${obj.status}: ${obj.detail}`)}'>◉ ${escapeHtml(obj.status.toLowerCase())}</span>`
         : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${objMark}${reps}</td>`;
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${objMark}${reps}${ungraded}</td>`;
     });
     html += "</tr>";
   }
@@ -389,7 +391,7 @@ function sparkline(runs) {
 function renderTrends(data) {
   const box = document.getElementById("trends");
   if (!data.models.length) { box.innerHTML = `<div class="empty-state">No runs yet.</div>`; return; }
-  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", "NOT-MEASURED": "∅" };
+  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", UNGRADED: "?", "NOT-MEASURED": "∅" };
   let html = "";
   for (const m of data.models) {
     const last = m.runs[m.runs.length - 1].grade;

package/dist/cli.js

@@ -2,7 +2,7 @@
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
@@ -44,7 +44,7 @@ function parseArgs(argv) {
 function assertNoRetiredFlags(command, args) {
     const retired = command === "run"
         ? ["affected"]
-        : command === "grade"
+        : command === "grade" || command === "regrade"
             ? ["auto-rejudge", "secondary-judge", "tie-break-judge"]
             : [];
     const found = retired.find((flag) => Object.hasOwn(args.flags, flag));
@@ -138,7 +138,9 @@ async function cmdList(args) {
             const seeded = spec.scenarios.filter((x) => x.mode === "seeded").length;
             const seededNote = seeded ? `, ${seeded} seeded` : "";
             const sourceNote = specs ? `; spec: --${s.specSource}` : "";
-            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote}${sourceNote})`);
+            const ungraded = collectReport(dirname(dirname(s.specPath))).columns.reduce((total, column) => total + Object.values(column.cells).reduce((n, cell) => n + (cell.ungraded_reps ?? 0), 0), 0);
+            const ungradedNote = ungraded ? `; latest results: ${ungraded} UNGRADED rep${ungraded === 1 ? "" : "s"}` : "";
+            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote}${sourceNote}${ungradedNote})`);
         }
         catch (e) {
             console.log(`  ✗ ${s.name}  INVALID: ${e instanceof Error ? e.message : e}`);
@@ -253,7 +255,7 @@ export async function cmdRun(args, adapterOverride) {
 export async function cmdGrade(args, adapterOverride) {
     const runDir = args._[0];
     if (!runDir)
-        throw new Error("usage: skill-harness grade <run-dir> [--judge prov:model] [--suspect-only]");
+        throw new Error("usage: skill-harness grade <run-dir> [--judge prov:model] [--suspect-only | --unparsed-only]");
     if (!existsSync(runDir)) {
         throw new Error(`run dir not found: ${resolve(runDir)} (relative paths resolve against the cwd)`);
     }
@@ -266,8 +268,13 @@ export async function cmdGrade(args, adapterOverride) {
     // an explicit --judge flag still wins; absent or retired recorded judges use
     // the current Pi default.
     const judgeFlag = flagStr(args, "judge");
+    const onlyUnparsed = flagBool(args, "unparsed-only");
+    if (onlyUnparsed && judgeFlag !== undefined)
+        throw new Error("--unparsed-only uses the recorded judge; --judge cannot be combined with it");
     const resolvedRecordedJudge = judgeFlag ? undefined : recordedJudgeOrDefault(prev?.judge);
     const judge = judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge;
+    if (onlyUnparsed && resolvedRecordedJudge?.migratedFrom)
+        throw new Error("--unparsed-only cannot use this retired recorded judge; use grade to migrate it explicitly");
     if (resolvedRecordedJudge?.migratedFrom) {
         console.error(`skill-harness: recorded judge ${resolvedRecordedJudge.migratedFrom.provider}:${resolvedRecordedJudge.migratedFrom.model} was removed; re-judging through Pi with ${judge.provider}:${judge.model}`);
     }
@@ -287,7 +294,7 @@ export async function cmdGrade(args, adapterOverride) {
         console.error(stale);
     const results = await regradeRun({
         runDir, spec, adapter, judge, specDir: testsDir, now: nowIso,
-        onlySuspect: flagBool(args, "suspect-only"),
+        onlySuspect: flagBool(args, "suspect-only"), onlyUnparsed,
     });
     for (const s of results.scenarios) {
         console.log(`  ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
@@ -731,6 +738,7 @@ export function help() {
                      [--arm <name>]  measure under a named arm from <skills-root>/tests/arms.yaml
                                      (loads its extensions, seeds pi-daddy definitions, tags the run dir)
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
+  regrade <run-dir>  --unparsed-only                      re-judge ERROR criterion reps with recorded judge
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (${free("rescore")})
   regate <run-dir>...  [--judge prov:model]     re-evaluate saved gates (no subject call; judges fail→pass reps)
   restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (${free("restamp")}; one-time migration)
@@ -761,7 +769,8 @@ export async function main(argv) {
     assertNoRetiredFlags(cmd, args);
     switch (cmd) {
         case "run": return cmdRun(args);
-        case "grade": return cmdGrade(args);
+        case "grade":
+        case "regrade": return cmdGrade(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
         case "restamp": return cmdRestamp(args);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.22.0",
+    "version": "0.22.4",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.22.0",
-        "@skill-harness/adapters": "0.22.0"
+        "@skill-harness/core": "0.22.4",
+        "@skill-harness/adapters": "0.22.4"
     }
 }