@skill-harness/cli 0.29.0 → 0.30.0

package/README.md

@@ -9,7 +9,7 @@ through Pi.
 Most users should install the `skill-harness` meta-package (`npm i -g
 skill-harness`) instead of this package directly.
 
-Fresh subject execution requires **exactly Pi 1.0.4**. Different or unavailable
+Fresh subject execution supports **Pi 1.0.4 and 1.1.0**. Other or unavailable
 versions produce execution ERROR before a subject call. Retained results remain
 readable; the historical Pi 0.83.0 raw parser is not current qualification.
 
@@ -19,12 +19,30 @@ readable; the historical Pi 0.83.0 raw parser is not current qualification.
 npm i -g @skill-harness/cli
 ```
 
+## Create a skill in Pi
+
+Install the `skill-harness` Pi package, select a model, then use
+`/skill-harness create <your idea>`. Authoring runs in Pi and asks one necessary
+question at a time; the standalone CLI does not substitute a direct model API.
+The workflow previews a skill, optional pi-daddy wrapper, and evals in the same
+or a separate repository before applying files. Generated skills need only Pi;
+the wrapper is optional and principal-pi-skills is not required.
+
+The CLI supports the subsequent offline preflight, reviewed judge calibration,
+scenario execution and saved-result diagnosis. See
+[EVAL-AUTHORING.md](https://github.com/mojomanyana/skill-harness/blob/main/docs/EVAL-AUTHORING.md)
+for the complete workflow and its evidence limits.
+
 ## Commands
 
 | command | does |
 |---|---|
 | `skill-harness list --skills <root>` | discovered skills + spec status |
 | `skill-harness lint <skill\|all> --skills <root>` | validate specs/fixtures; CI gate |
+| `skill-harness preflight <skill\|all> --skills <root> [--specs <root>]` | inspect eval configuration and discoverability offline |
+| `skill-harness calibrate <manifest.json>` | validate selected calibration evidence offline |
+| `skill-harness calibrate <manifest.json> --run --judge p:m --out NEW_DIR` | judge reviewed controls through Pi; preserve a new result directory |
+| `skill-harness diagnose <run-dir> --skills <root> [--specs <root>]` | inspect saved evidence without changing historical scores |
 | `skill-harness run <skill\|all> --skills <root> [--model p:m ...] [--judge p:m]` | run scenarios, grade, score |
 | `skill-harness grade <run-dir> [--judge p:m]` | re-grade saved transcripts |
 | `skill-harness rescore <run-dir>...` | re-apply current score thresholds offline |

package/dist/cli.d.ts

@@ -51,4 +51,7 @@ export declare function cmdLint(args: Args): Promise<void>;
  * prints none.
  */
 export declare function help(): string;
+export declare function cmdPreflight(args: Args): Promise<void>;
+export declare function cmdDiagnose(args: Args): Promise<void>;
+export declare function cmdCalibrate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function main(argv: string[]): Promise<void>;

package/dist/cli.js

@@ -2,7 +2,7 @@
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, diagnoseRun, loadCalibration, runCalibration, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 import { cmdDecision } from "./decision.js";
@@ -747,6 +747,10 @@ export function help() {
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
   init   <skill>     --skills <root> [--force]     scaffold a commented template spec (${free("init")})
+  preflight <skill|all> --skills <root> [--specs <root>] eval checks (${free("preflight")})
+  calibrate <manifest.json> [--run --judge p:m --out NEW_DIR] validate or judge reviewed controls
+  diagnose <run-dir> --skills <root> [--specs <root>] inspect saved evidence (${free("diagnose")})
+  In Pi: /skill-harness create <idea>             guided skill + optional agent + eval creation
   suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
   list   --skills <root>                        discovered skills + spec status (${free("list")})
   lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (${free("lint")}; CI gate; exits non-zero on findings)
@@ -766,6 +770,51 @@ defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=p
   the judge default runs through Pi on your ChatGPT subscription (\`openai-codex\`), not a metered
   API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
 }
+export async function cmdPreflight(args) {
+    const { skills, specs } = roots(args);
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness preflight <skill|all> --skills <root> [--specs <root>]");
+    const selected = target === "all" ? discoverUsable(skills, specs).filter(s => s.hasSpec) : [resolveSkill(skills, target, specs)];
+    if (!selected.length)
+        throw new Error("No specifications found; in Pi use /skill-harness create to author a skill and its evals.");
+    const findings = selected.flatMap(s => lintSkill(s.dir, s.testsDir));
+    const errors = findings.filter(failsGate);
+    console.log(JSON.stringify({ status: errors.length ? "blocked" : "checked", providerCalls: 0, semanticValidity: "requires-reviewed-controls", findings }, null, 2));
+    if (errors.length)
+        process.exitCode = 1;
+}
+export async function cmdDiagnose(args) {
+    if (!args._[0])
+        throw new Error("usage: skill-harness diagnose <run-dir> --skills <root> [--specs <root>]");
+    const runDir = resolve(args._[0]);
+    const recorded = readResults(runDir);
+    const { skills, specs } = roots(args);
+    const selected = resolveSkill(skills, recorded.skill, specs);
+    console.log(JSON.stringify(diagnoseRun(runDir, selected.dir, selected.specPath), null, 2));
+}
+export async function cmdCalibrate(args, adapterOverride) {
+    if (!args._[0])
+        throw new Error("usage: skill-harness calibrate <manifest.json> [--run --judge provider:model --out NEW_DIR]");
+    const calibration = loadCalibration(resolve(args._[0]));
+    if (!flagBool(args, "run")) {
+        console.log(JSON.stringify({ status: calibration.ready ? "validated" : "needs-review", purpose: "judge-calibration-only", providerCalls: 0, cases: calibration.controls.length, issues: calibration.issues }));
+        if (!calibration.ready)
+            process.exitCode = 1;
+        return;
+    }
+    const judgeToken = flagStr(args, "judge");
+    const output = flagStr(args, "out");
+    if (!judgeToken || !output)
+        throw new Error("Calibration --run needs an explicitly selected --judge and a new --out directory.");
+    const adapter = adapterOverride ?? getAdapter("pi");
+    if (!await adapter.available())
+        throw new Error("Pi is required for judge calibration. Configure the selected model in Pi first.");
+    const result = await runCalibration(calibration, { adapter, judge: parseModelRef(judgeToken), output: resolve(output), allowMeteredJudge: flagBool(args, "allow-metered-judge"), onProgress: p => console.log(`Judging control ${p.index + 1}/${p.total}: ${p.id}`) });
+    console.log(JSON.stringify(result, null, 2));
+    if (result.status !== "matched")
+        process.exitCode = 1;
+}
 export async function main(argv) {
     const cmd = argv[0];
     if (cmd === "decision")
@@ -773,6 +822,11 @@ export async function main(argv) {
     const args = parseArgs(argv.slice(1));
     assertNoRetiredFlags(cmd, args);
     switch (cmd) {
+        case "create":
+        case "author": throw new Error("Skill creation uses your current Pi model. Start Pi with skill-harness installed, then use /skill-harness create followed by your idea; it asks one necessary question at a time.");
+        case "preflight": return cmdPreflight(args);
+        case "diagnose": return cmdDiagnose(args);
+        case "calibrate": return cmdCalibrate(args);
         case "run": return cmdRun(args);
         case "grade":
         case "regrade": return cmdGrade(args);

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, recordedJudgeOrDefault, assertJudgeAllowed, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, recordedJudgeOrDefault, assertJudgeAllowed, parseModelRef, resolveJudgeReferences, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -145,10 +145,16 @@ export async function serveReview(opts) {
                 }
                 const threshold = effectiveThreshold(prev, scenario);
                 try {
+                    // The prompt and refreshed rubric must name the same selected bytes,
+                    // even when a reference is edited while a multi-repetition judge runs.
+                    const specDir = dirname(specPath);
+                    const references = resolveJudgeReferences(scenario, specDir);
                     const rr = await regradeScenario({
                         runDir: column.runDir, spec, scenario, adapter, judge: resolvedRecordedJudge.judge,
-                        specDir: dirname(specPath), threshold, mode: results.mode,
+                        specDir, threshold, mode: results.mode, references,
                         expectedReps: prev.reps ?? 1,
+                        prior: prev, piVersion: results.harness_cli_version,
+                        ...(results.model.includes(":") ? { subject: parseModelRef(results.model) } : {}),
                     });
                     const merged = results.scenarios.map((s) => {
                         if (s.id !== body.scenarioId)
@@ -174,7 +180,7 @@ export async function serveReview(opts) {
                         // retired the staleness gate for any run re-judged from the UI. Carried,
                         // with the one `rubric:` key this re-judge actually applied refreshed —
                         // the same doctrine `grade` follows (see refreshRubricHashes).
-                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
+                        source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId], specDir, { [body.scenarioId]: references }),
                         source_hash_roots: results.source_hash_roots,
                     }, scoreContextFor(results, spec));
                     ensureResultsGitignore(join(opts.skillDir, "tests", "results"));

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.29.0",
+    "version": "0.30.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.29.0",
-        "@skill-harness/adapters": "0.29.0"
+        "@skill-harness/core": "0.30.0",
+        "@skill-harness/adapters": "0.30.0"
     }
 }