@skill-harness/cli 0.22.3 → 0.23.0

package/assets/report.grade.js

@@ -55,6 +55,7 @@ export function gradeColumn(col, shipBar, critical) {
   let suspect = 0;
   let errors = 0;
   let notMeasured = 0;
+  let ungraded = 0;
 
   for (const id of Object.keys(col.cells)) {
     const cell = col.cells[id];
@@ -77,12 +78,13 @@ export function gradeColumn(col, shipBar, critical) {
       passed++;
       continue;
     }
+    if (verdict === "UNGRADED") ungraded++;
     if (critical.includes(id)) criticalFails++;
     if (/^B/i.test(id)) bFails++;
   }
 
   if (col.partial === true) {
-    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured };
+    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured, ungraded };
   }
 
   const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
@@ -98,7 +100,7 @@ export function gradeColumn(col, shipBar, critical) {
     errors === 0 &&
     notMeasured === 0;
 
-  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured };
+  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured, ungraded };
 }
 
 /**
@@ -114,7 +116,7 @@ export function gradeColumn(col, shipBar, critical) {
  */
 export function liftClass(liftCell, greenCell) {
   const conclusive = (verdict, suspect) =>
-    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
+    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS" && verdict !== "UNGRADED";
 
   const redOk = conclusive(liftCell.red, liftCell.redSuspect);
   const greenOk = conclusive(effective(greenCell), !!greenCell.suspect && !greenCell.override);

package/assets/report.template.html

@@ -25,6 +25,7 @@
   .cell.PASS { background: rgba(31,139,76,.22); }
   .cell.FAIL { background: rgba(192,57,43,.24); }
   .cell.ERROR { background: rgba(107,114,128,.22); }
+  .cell.UNGRADED { background: rgba(180,83,9,.18); }
   .cell.NOT-MEASURED { background: rgba(180,83,9,.18); }
   .cell.empty { color: var(--dim); cursor: default; font-weight: 400; }
   .cell .ov { display: block; font-size: 11px; color: var(--over); font-weight: 600; }
@@ -229,6 +230,7 @@ function render() {
         ? `<span class='ov unsaved'>unsaved</span>`
         : (cell.override ? `<span class='ov'>override</span>` : "");
       const misfired = cell.clean != null && cell.clean < cell.reps ? ` · ${cell.reps - cell.clean} misfired` : "";
+      const ungraded = cell.ungraded_reps ? `<span class='reps'>${cell.ungraded_reps} UNGRADED rep${cell.ungraded_reps === 1 ? "" : "s"} (not judged FAIL)</span>` : "";
       const reps = cell.reps ? `<span class='reps'>${cell.passes}/${cell.clean}${misfired}${cell.flakiness ? ` · flaky ${cell.flakiness.toFixed(2)}` : ""}</span>` : "";
       const suspectBadge = cell.suspect && !cell.override ? `<span class='ov suspect'>suspect</span>` : "";
       // Per-cell lift marker: only the two classes that mean the skill changed
@@ -255,7 +257,7 @@ function render() {
       const objMark = obj
         ? `<span class='lm obj-${obj.status.toLowerCase()}' title='${escapeHtml(`objective ${obj.status}: ${obj.detail}`)}'>◉ ${escapeHtml(obj.status.toLowerCase())}</span>`
         : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${objMark}${reps}</td>`;
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${stabMark}${objMark}${reps}${ungraded}</td>`;
     });
     html += "</tr>";
   }
@@ -389,7 +391,7 @@ function sparkline(runs) {
 function renderTrends(data) {
   const box = document.getElementById("trends");
   if (!data.models.length) { box.innerHTML = `<div class="empty-state">No runs yet.</div>`; return; }
-  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", "NOT-MEASURED": "∅" };
+  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", UNGRADED: "?", "NOT-MEASURED": "∅" };
   let html = "";
   for (const m of data.models) {
     const last = m.runs[m.runs.length - 1].grade;

package/dist/cli.js

@@ -2,7 +2,7 @@
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
@@ -138,7 +138,9 @@ async function cmdList(args) {
             const seeded = spec.scenarios.filter((x) => x.mode === "seeded").length;
             const seededNote = seeded ? `, ${seeded} seeded` : "";
             const sourceNote = specs ? `; spec: --${s.specSource}` : "";
-            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote}${sourceNote})`);
+            const ungraded = collectReport(dirname(dirname(s.specPath))).columns.reduce((total, column) => total + Object.values(column.cells).reduce((n, cell) => n + (cell.ungraded_reps ?? 0), 0), 0);
+            const ungradedNote = ungraded ? `; latest results: ${ungraded} UNGRADED rep${ungraded === 1 ? "" : "s"}` : "";
+            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote}${sourceNote}${ungradedNote})`);
         }
         catch (e) {
             console.log(`  ✗ ${s.name}  INVALID: ${e instanceof Error ? e.message : e}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.22.3",
+    "version": "0.23.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.22.3",
-        "@skill-harness/adapters": "0.22.3"
+        "@skill-harness/core": "0.23.0",
+        "@skill-harness/adapters": "0.23.0"
     }
 }