@skill-harness/cli 0.29.0 → 0.31.0

package/README.md

@@ -9,7 +9,7 @@ through Pi.
 Most users should install the `skill-harness` meta-package (`npm i -g
 skill-harness`) instead of this package directly.
 
-Fresh subject execution requires **exactly Pi 1.0.4**. Different or unavailable
+Fresh subject execution supports **Pi 1.0.4 and 1.1.0**. Other or unavailable
 versions produce execution ERROR before a subject call. Retained results remain
 readable; the historical Pi 0.83.0 raw parser is not current qualification.
 
@@ -19,12 +19,30 @@ readable; the historical Pi 0.83.0 raw parser is not current qualification.
 npm i -g @skill-harness/cli
 ```
 
+## Create a skill in Pi
+
+Install the `skill-harness` Pi package, select a model, then use
+`/skill-harness create <your idea>`. Authoring runs in Pi and asks one necessary
+question at a time; the standalone CLI does not substitute a direct model API.
+The workflow previews a skill, optional pi-daddy wrapper, and evals in the same
+or a separate repository before applying files. Generated skills need only Pi;
+the wrapper is optional and principal-pi-skills is not required.
+
+The CLI supports the subsequent offline preflight, reviewed judge calibration,
+scenario execution and saved-result diagnosis. See
+[EVAL-AUTHORING.md](https://github.com/mojomanyana/skill-harness/blob/main/docs/EVAL-AUTHORING.md)
+for the complete workflow and its evidence limits.
+
 ## Commands
 
 | command | does |
 |---|---|
 | `skill-harness list --skills <root>` | discovered skills + spec status |
 | `skill-harness lint <skill\|all> --skills <root>` | validate specs/fixtures; CI gate |
+| `skill-harness preflight <skill\|all> --skills <root> [--specs <root>]` | inspect eval configuration and discoverability offline |
+| `skill-harness calibrate <manifest.json>` | validate selected calibration evidence offline |
+| `skill-harness calibrate <manifest.json> --run --judge p:m --out NEW_DIR` | judge reviewed controls through Pi; preserve a new result directory |
+| `skill-harness diagnose <run-dir> --skills <root> [--specs <root>]` | inspect saved evidence without changing historical scores |
 | `skill-harness run <skill\|all> --skills <root> [--model p:m ...] [--judge p:m]` | run scenarios, grade, score |
 | `skill-harness grade <run-dir> [--judge p:m]` | re-grade saved transcripts |
 | `skill-harness rescore <run-dir>...` | re-apply current score thresholds offline |

package/dist/cli.d.ts

@@ -51,4 +51,7 @@ export declare function cmdLint(args: Args): Promise<void>;
  * prints none.
  */
 export declare function help(): string;
+export declare function cmdPreflight(args: Args): Promise<void>;
+export declare function cmdDiagnose(args: Args): Promise<void>;
+export declare function cmdCalibrate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function main(argv: string[]): Promise<void>;

package/dist/cli.js

@@ -2,7 +2,7 @@
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, diagnoseRun, loadCalibration, runCalibration, planCalibration, forkSavedRun, withProcessScope, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 import { cmdDecision } from "./decision.js";
@@ -11,7 +11,7 @@ const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
 // before, and the pi extension's copy could disagree with this one.
 const DEFAULT_SUGGEST_MODEL = "openai-codex:gpt-5.6-sol";
-const REPEATABLE = new Set(["model", "turn", "check"]);
+const REPEATABLE = new Set(["model", "turn", "check", "reuse"]);
 function parseArgs(argv) {
     const _ = [];
     const flags = {};
@@ -65,6 +65,15 @@ export function flagBool(args, key) {
     const v = args.flags[key];
     return v === true || v === "true" || v === "1";
 }
+function selectedIds(args) {
+    const raw = flagStr(args, "only");
+    if (raw === undefined)
+        return undefined;
+    const ids = raw.split(",").map(s => s.trim());
+    if (ids.some(id => !id) || new Set(ids).size !== ids.length)
+        throw new Error("--only requires a nonempty, unique comma-separated ID list");
+    return ids;
+}
 function resolveModels(args) {
     const models = [...(args.multi.model ?? [])];
     const file = flagStr(args, "models");
@@ -174,8 +183,7 @@ export async function cmdRun(args, adapterOverride) {
     // not on PATH" for a plain flag typo, sending the reader off to fix an install
     // that was never the problem. Free, offline validation first — it is also why
     // CI, which has no `pi`, could not test this refusal at all.
-    const onlyRaw = flagStr(args, "only");
-    const only = onlyRaw ? onlyRaw.split(",").map((x) => x.trim()).filter(Boolean) : undefined;
+    const only = selectedIds(args);
     const harnessName = flagStr(args, "harness", "pi");
     const adapter = adapterOverride ?? getAdapter(harnessName);
     if (!(await adapter.available()))
@@ -254,15 +262,16 @@ export async function cmdRun(args, adapterOverride) {
         process.exitCode = 1;
 }
 export async function cmdGrade(args, adapterOverride) {
-    const runDir = args._[0];
+    const only = selectedIds(args);
+    let runDir = args._[0];
     if (!runDir)
         throw new Error("usage: skill-harness grade <run-dir> [--judge prov:model] [--suspect-only | --unparsed-only]");
     if (!existsSync(runDir)) {
         throw new Error(`run dir not found: ${resolve(runDir)} (relative paths resolve against the cwd)`);
     }
     // spec lives at <runDir>/../../../specification.yaml  (results/<tag>/<ts> -> tests/)
-    const testsDir = dirname(dirname(dirname(runDir)));
-    const specPath = join(testsDir, "specification.yaml");
+    const specPath = flagStr(args, "spec") ? resolve(flagStr(args, "spec")) : join(dirname(dirname(dirname(runDir))), "specification.yaml");
+    const testsDir = dirname(specPath);
     const spec = loadSpec(specPath);
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
     // Re-judge with the run's RECORDED judge + harness (parity with /rejudge) —
@@ -293,9 +302,17 @@ export async function cmdGrade(args, adapterOverride) {
     const stale = downgradeWarning(dirname(testsDir));
     if (stale)
         console.error(stale);
+    const output = flagStr(args, "out");
+    if (output !== undefined) {
+        if (!output.trim())
+            throw new Error("--out requires a new directory");
+        runDir = forkSavedRun(runDir, output);
+        console.log(`Reusing saved answers in ${runDir}; original run preserved.`);
+    }
     const results = await regradeRun({
         runDir, spec, adapter, judge, specDir: testsDir, now: nowIso,
         onlySuspect: flagBool(args, "suspect-only"), onlyUnparsed,
+        only,
     });
     for (const s of results.scenarios) {
         console.log(`  ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
@@ -734,11 +751,12 @@ export function help() {
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
-                     [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
+                     [--canary]  green only: one instructed-loading probe; autonomous selection stays untested
                      [--structured]  request structured capture (Pi uses it by default for subject usage)
                      [--arm <name>]  measure under a named arm from <skills-root>/tests/arms.yaml
                                      (loads its extensions, seeds pi-daddy definitions, tags the run dir)
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
+                     [--only A1,A2] [--out NEW_DIR] [--spec path] preserve originals and judge selected saved answers
   regrade <run-dir>  --unparsed-only                      re-judge ERROR criterion reps with recorded judge
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (${free("rescore")})
   regate <run-dir>...  [--judge prov:model]     re-evaluate saved gates (no subject call; judges fail→pass reps)
@@ -747,6 +765,13 @@ export function help() {
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
   init   <skill>     --skills <root> [--force]     scaffold a commented template spec (${free("init")})
+  preflight <skill|all> --skills <root> [--specs <root>] eval checks (${free("preflight")})
+  calibrate <manifest.json> [--run --judge p:m --out NEW_DIR] validate or judge reviewed controls
+                     [--plan --judge p:m] [--only c1,c2] [--reuse PRIOR_DIR ...] exact historical reuse, no new subject calls
+  diagnose <run-dir> --skills <root> [--specs <root>] inspect saved evidence (${free("diagnose")})
+  Paid commands accept --max-memory-mb N / --max-write-mb N for sampled Linux process limits.
+  run/grade/calibrate save bounded private resources-*.json; unsupported host metrics stay unknown.
+  In Pi: /skill-harness create <idea>             guided skill + optional agent + eval creation
   suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
   list   --skills <root>                        discovered skills + spec status (${free("list")})
   lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (${free("lint")}; CI gate; exits non-zero on findings)
@@ -762,17 +787,105 @@ export function help() {
 defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=pi
   green and force are both scored; red is the unscored baseline. green delivery depends on the
   harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
-  so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
+  --mode force supplies the skill body; --canary checks instructed loading separately from autonomous selection.
   the judge default runs through Pi on your ChatGPT subscription (\`openai-codex\`), not a metered
   API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
 }
+export async function cmdPreflight(args) {
+    const { skills, specs } = roots(args);
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness preflight <skill|all> --skills <root> [--specs <root>]");
+    const selected = target === "all" ? discoverUsable(skills, specs).filter(s => s.hasSpec) : [resolveSkill(skills, target, specs)];
+    if (!selected.length)
+        throw new Error("No specifications found; in Pi use /skill-harness create to author a skill and its evals.");
+    const findings = selected.flatMap(s => lintSkill(s.dir, s.testsDir));
+    const errors = findings.filter(failsGate);
+    console.log(JSON.stringify({ status: errors.length ? "blocked" : "checked", providerCalls: 0, semanticValidity: "requires-reviewed-controls", findings }, null, 2));
+    if (errors.length)
+        process.exitCode = 1;
+}
+export async function cmdDiagnose(args) {
+    if (!args._[0])
+        throw new Error("usage: skill-harness diagnose <run-dir> --skills <root> [--specs <root>]");
+    const runDir = resolve(args._[0]);
+    const recorded = readResults(runDir);
+    const { skills, specs } = roots(args);
+    const selected = resolveSkill(skills, recorded.skill, specs);
+    console.log(JSON.stringify(diagnoseRun(runDir, selected.dir, selected.specPath), null, 2));
+}
+export async function cmdCalibrate(args, adapterOverride) {
+    if (!args._[0])
+        throw new Error("usage: skill-harness calibrate <manifest.json> [--run --judge provider:model --out NEW_DIR]");
+    const calibration = loadCalibration(resolve(args._[0]));
+    const judgeToken = flagStr(args, "judge");
+    const only = selectedIds(args);
+    const reuse = (args.multi.reuse ?? []).map(p => { if (!p.trim())
+        throw new Error("--reuse requires a prior calibration directory"); return resolve(p); });
+    if (flagBool(args, "plan")) {
+        if (!judgeToken)
+            throw new Error("Calibration --plan needs an explicit --judge; it makes zero provider calls.");
+        const plan = planCalibration(calibration, { judge: parseModelRef(judgeToken), only, reuse });
+        console.log(JSON.stringify({ ...plan, providerCalls: 0 }, null, 2));
+        if (!plan.ready)
+            process.exitCode = 1;
+        return;
+    }
+    if (!flagBool(args, "run")) {
+        console.log(JSON.stringify({ status: calibration.ready ? "validated" : "needs-review", purpose: "judge-calibration-only", executionGates: "not-evaluated", providerCalls: 0, cases: calibration.controls.length, issues: calibration.issues }));
+        if (!calibration.ready)
+            process.exitCode = 1;
+        return;
+    }
+    const output = flagStr(args, "out");
+    if (!judgeToken || !output)
+        throw new Error("Calibration --run needs an explicitly selected --judge and a new --out directory.");
+    const adapter = adapterOverride ?? getAdapter("pi");
+    if (!await adapter.available())
+        throw new Error("Pi is required for judge calibration. Configure the selected model in Pi first.");
+    const result = await runCalibration(calibration, { adapter, judge: parseModelRef(judgeToken), only, reuse, output: resolve(output), allowMeteredJudge: flagBool(args, "allow-metered-judge"), onProgress: p => console.log(`Judging control ${p.index + 1}/${p.total}: ${p.id}`) });
+    console.log(JSON.stringify(result, null, 2));
+    if (result.status !== "matched")
+        process.exitCode = 1;
+}
 export async function main(argv) {
+    if (!["run", "grade", "regrade", "calibrate", "regate", "suggest"].includes(argv[0]))
+        return dispatch(argv);
+    const args = parseArgs(argv.slice(1));
+    const limit = (name) => {
+        const raw = flagStr(args, name);
+        if (raw === undefined)
+            return undefined;
+        const number = Number(raw);
+        if (!Number.isFinite(number) || number <= 0 || !Number.isSafeInteger(number * 1048576))
+            throw new Error(`--${name} requires a positive MiB value`);
+        return number * 1048576;
+    };
+    const controller = new AbortController();
+    const interrupt = () => { process.exitCode = 130; controller.abort(new Error("SIGINT")); };
+    const terminate = () => { process.exitCode = 143; controller.abort(new Error("SIGTERM")); };
+    process.once("SIGINT", interrupt);
+    process.once("SIGTERM", terminate);
+    try {
+        await withProcessScope({ signal: controller.signal, maxRssBytes: limit("max-memory-mb"), maxWriteBytes: limit("max-write-mb") }, () => dispatch(argv));
+    }
+    finally {
+        process.removeListener("SIGINT", interrupt);
+        process.removeListener("SIGTERM", terminate);
+    }
+}
+async function dispatch(argv) {
     const cmd = argv[0];
     if (cmd === "decision")
         return cmdDecision(argv.slice(1));
     const args = parseArgs(argv.slice(1));
     assertNoRetiredFlags(cmd, args);
     switch (cmd) {
+        case "create":
+        case "author": throw new Error("Skill creation uses your current Pi model. Start Pi with skill-harness installed, then use /skill-harness create followed by your idea; it asks one necessary question at a time.");
+        case "preflight": return cmdPreflight(args);
+        case "diagnose": return cmdDiagnose(args);
+        case "calibrate": return cmdCalibrate(args);
         case "run": return cmdRun(args);
         case "grade":
         case "regrade": return cmdGrade(args);
@@ -813,7 +926,7 @@ export async function main(argv) {
 if (!process.env.VITEST) {
     main(process.argv.slice(2)).catch((e) => {
         console.error(`error: ${e instanceof Error ? e.message : e}`);
-        process.exitCode = 1;
+        process.exitCode ||= 1;
     });
 }
 //# sourceMappingURL=cli.js.map

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, recordedJudgeOrDefault, assertJudgeAllowed, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, recordedJudgeOrDefault, assertJudgeAllowed, parseModelRef, resolveJudgeReferences, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -145,10 +145,16 @@ export async function serveReview(opts) {
                 }
                 const threshold = effectiveThreshold(prev, scenario);
                 try {
+                    // The prompt and refreshed rubric must name the same selected bytes,
+                    // even when a reference is edited while a multi-repetition judge runs.
+                    const specDir = dirname(specPath);
+                    const references = resolveJudgeReferences(scenario, specDir);
                     const rr = await regradeScenario({
                         runDir: column.runDir, spec, scenario, adapter, judge: resolvedRecordedJudge.judge,
-                        specDir: dirname(specPath), threshold, mode: results.mode,
+                        specDir, threshold, mode: results.mode, references,
                         expectedReps: prev.reps ?? 1,
+                        prior: prev, piVersion: results.harness_cli_version,
+                        ...(results.model.includes(":") ? { subject: parseModelRef(results.model) } : {}),
                     });
                     const merged = results.scenarios.map((s) => {
                         if (s.id !== body.scenarioId)
@@ -174,7 +180,7 @@ export async function serveReview(opts) {
                         // retired the staleness gate for any run re-judged from the UI. Carried,
                         // with the one `rubric:` key this re-judge actually applied refreshed —
                         // the same doctrine `grade` follows (see refreshRubricHashes).
-                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
+                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId], specDir, { [body.scenarioId]: references }),
                         source_hash_roots: results.source_hash_roots,
                     }, scoreContextFor(results, spec));
                     ensureResultsGitignore(join(opts.skillDir, "tests", "results"));

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.29.0",
+    "version": "0.31.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.29.0",
-        "@skill-harness/adapters": "0.29.0"
+        "@skill-harness/core": "0.31.0",
+        "@skill-harness/adapters": "0.31.0"
     }
 }