@skill-harness/cli 0.18.0 → 0.20.0

package/README.md

@@ -3,7 +3,8 @@
 The command surface for [skill-harness](https://github.com/mojomanyana/skill-harness) —
 a test/optimize loop for agent skills: run spec'd scenarios on the `pi` harness,
 LLM-judge each transcript, score against a ship bar, review, and re-run to
-measure a `SKILL.md` edit.
+measure a `SKILL.md` edit. Subject, judge, and spec-drafting model calls all run
+through Pi.
 
 Most users should install the `skill-harness` meta-package (`npm i -g
 skill-harness`) instead of this package directly.
@@ -22,8 +23,15 @@ npm i -g @skill-harness/cli
 | `skill-harness lint <skill\|all> --skills <root>` | validate specs/fixtures; CI gate |
 | `skill-harness run <skill\|all> --skills <root> [--model p:m ...] [--judge p:m]` | run scenarios, grade, score |
 | `skill-harness grade <run-dir> [--judge p:m]` | re-grade saved transcripts |
+| `skill-harness rescore <run-dir>...` | re-apply current score thresholds offline |
+| `skill-harness regate <run-dir>...` | re-evaluate saved objective gates |
+| `skill-harness restamp <skill\|all> --skills <root> --from <ref>` | migrate matching model-visible digests |
+| `skill-harness stability <skill\|all> --skills <root>` | report run-over-run verdict flips |
 | `skill-harness review <skill> --skills <root> [--port N]` | serve the interactive review UI |
 | `skill-harness add-test <skill> --skills <root> --id ID --title T ...` | scaffold a new scenario |
+| `skill-harness init <skill> --skills <root>` | write an offline spec template |
+| `skill-harness suggest <skill> --skills <root>` | draft a spec through Pi |
+| `skill-harness coverage <skill\|all> --skills <root>` | report declared instruction coverage |
 
 ## More
 

package/dist/cli.js

@@ -2,14 +2,14 @@
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve, relative } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 // The judge default lives in core (`defaultJudge()`), which resolves
 // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
 // before, and the pi extension's copy could disagree with this one.
-const DEFAULT_SUGGEST_MODEL = "claude-code:claude-opus-4-8";
+const DEFAULT_SUGGEST_MODEL = "openai-codex:gpt-5.6-sol";
 const REPEATABLE = new Set(["model", "turn", "check"]);
 function parseArgs(argv) {
     const _ = [];
@@ -236,14 +236,17 @@ export async function cmdGrade(args, adapterOverride) {
     const spec = loadSpec(specPath);
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
     // Re-judge with the run's RECORDED judge + harness (parity with /rejudge) —
-    // an explicit --judge flag still wins; with no prior results, fall back to
-    // the CLI default.
+    // an explicit --judge flag still wins; absent or retired recorded judges use
+    // the current Pi default.
     const judgeFlag = flagStr(args, "judge");
-    const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(defaultJudge()));
-    // A regrade reuses the judge the run RECORDED, so a run that names a metered judge
-    // bills on every later regrade with no flag typed anywhere. Latent rather than live
-    // in the reference corpus (all ~140 committed runs there record `claude-code`), but
-    // it is the one path where the cost decision was made by a file, not a person.
+    const resolvedRecordedJudge = judgeFlag ? undefined : recordedJudgeOrDefault(prev?.judge);
+    const judge = judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge;
+    if (resolvedRecordedJudge?.migratedFrom) {
+        console.error(`skill-harness: recorded judge ${resolvedRecordedJudge.migratedFrom.provider}:${resolvedRecordedJudge.migratedFrom.model} was removed; re-judging through Pi with ${judge.provider}:${judge.model}`);
+    }
+    // A regrade normally reuses the judge the run RECORDED, so a run that names a
+    // metered judge bills on every later regrade with no flag typed anywhere. It is the one path
+    // where the cost decision was made by a file, not a person.
     assertJudgeAllowed(judge, {
         source: judgeFlag ? "--judge" : prev?.judge ? "the run's recorded judge" : "the default judge",
         allowMetered: flagBool(args, "allow-metered-judge"),
@@ -312,7 +315,11 @@ export async function cmdRegate(args, adapterOverride) {
         const specPath = specPathForRunDir(runDir);
         const spec = loadSpec(specPath);
         const prev = readResults(runDir);
-        const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev.judge ?? parseModelRef(defaultJudge()));
+        const resolvedRecordedJudge = judgeFlag ? undefined : recordedJudgeOrDefault(prev.judge);
+        const judge = judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge;
+        if (resolvedRecordedJudge?.migratedFrom) {
+            console.error(`skill-harness: recorded judge ${resolvedRecordedJudge.migratedFrom.provider}:${resolvedRecordedJudge.migratedFrom.model} was removed; re-judging through Pi with ${judge.provider}:${judge.model}`);
+        }
         assertJudgeAllowed(judge, {
             source: judgeFlag ? "--judge" : "the run's recorded judge",
             allowMetered: flagBool(args, "allow-metered-judge"),
@@ -692,7 +699,7 @@ export function help() {
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
                      [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
-                     [--structured]  record subject tokens, cost and wall time (needs pi --mode json)
+                     [--structured]  request structured capture (Pi uses it by default for subject usage)
                      [--arm <name>]  measure under a named arm from <skills-root>/tests/arms.yaml
                                      (loads its extensions, seeds pi-daddy definitions, tags the run dir)
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
@@ -714,8 +721,8 @@ defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=p
   green and force are both scored; red is the unscored baseline. green delivery depends on the
   harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
   so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
-  the judge default is Opus on your Claude subscription (\`claude-code\` → \`claude -p\`), not a
-  metered API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
+  the judge default runs through Pi on your ChatGPT subscription (\`openai-codex\`), not a metered
+  API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
 }
 export async function main(argv) {
     const cmd = argv[0];

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, recordedJudgeOrDefault, assertJudgeAllowed, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -126,6 +126,17 @@ export async function serveReview(opts) {
                     res.end(JSON.stringify({ ok: false, error: `scenario ${body.scenarioId} is ${delivery?.status ?? "ERROR"}: delivery-gated evidence cannot be re-judged` }));
                     return;
                 }
+                const resolvedRecordedJudge = recordedJudgeOrDefault(results.judge);
+                try {
+                    assertJudgeAllowed(resolvedRecordedJudge.judge, {
+                        source: resolvedRecordedJudge.migratedFrom ? "the current default judge" : "the run's recorded judge",
+                    });
+                }
+                catch (e) {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: e instanceof Error ? e.message : String(e) }));
+                    return;
+                }
                 const adapter = opts.adapter ?? getAdapter(results.harness);
                 if (!(await adapter.available())) {
                     res.writeHead(400, { "content-type": "application/json" });
@@ -135,7 +146,7 @@ export async function serveReview(opts) {
                 const threshold = effectiveThreshold(prev, scenario);
                 try {
                     const rr = await regradeScenario({
-                        runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
+                        runDir: column.runDir, spec, scenario, adapter, judge: resolvedRecordedJudge.judge,
                         specDir: dirname(specPath), threshold, mode: results.mode,
                         expectedReps: prev.reps ?? 1,
                     });
@@ -147,7 +158,7 @@ export async function serveReview(opts) {
                     });
                     const written = writeResults(column.runDir, {
                         schema: results.schema, subject_invocations: results.subject_invocations,
-                        skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
+                        skill: results.skill, harness: results.harness, model: results.model, judge: resolvedRecordedJudge.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
                         partial: results.partial,
                         // Provenance survives a UI re-judge, same as it does through `grade`.
@@ -169,7 +180,13 @@ export async function serveReview(opts) {
                     const g = written.effective_grade;
                     appendJournal(column.runDir, { event: "score", ts: new Date().toISOString(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
                     res.writeHead(200, { "content-type": "application/json" });
-                    res.end(JSON.stringify({ ok: true, grade: g }));
+                    res.end(JSON.stringify({
+                        ok: true,
+                        grade: g,
+                        judgeMigration: resolvedRecordedJudge.migratedFrom
+                            ? `${resolvedRecordedJudge.migratedFrom.provider}:${resolvedRecordedJudge.migratedFrom.model} → ${resolvedRecordedJudge.judge.provider}:${resolvedRecordedJudge.judge.model}`
+                            : undefined,
+                    }));
                 }
                 catch (e) {
                     // regradeScenario (or the write/journal that follows) failed — surface the

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.18.0",
+    "version": "0.20.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.18.0",
-        "@skill-harness/adapters": "0.18.0"
+        "@skill-harness/core": "0.20.0",
+        "@skill-harness/adapters": "0.20.0"
     }
 }