@skill-harness/cli 0.11.0 → 0.12.0

package/README.md

@@ -24,6 +24,7 @@ npm i -g @skill-harness/cli
 | `skill-harness grade <run-dir> [--judge p:m]` | re-grade saved transcripts |
 | `skill-harness review <skill> --skills <root> [--port N]` | serve the interactive review UI |
 | `skill-harness add-test <skill> --skills <root> --id ID --title T ...` | scaffold a new scenario |
+| `skill-harness qualification prepare\|start\|status\|poll\|validate\|abort ...` | durable external-config qualification lifecycle |
 
 ## More
 

package/assets/report.grade.js

@@ -28,6 +28,7 @@ export function effective(cell) {
   // Mechanical evidence outranks a prose judge. Objective PASS deliberately
   // forces nothing — the checklist judge still decides the behavioral rubric.
   if (cell.objective && cell.objective.status === "ERROR") return "ERROR";
+  if (cell.objective && cell.objective.status === "NOT-MEASURED") return "NOT-MEASURED";
   if (cell.objective && cell.objective.status === "FAIL") return "FAIL";
   return cell.judge_verdict;
 }
@@ -53,6 +54,7 @@ export function gradeColumn(col, shipBar, critical) {
   let bFails = 0;
   let suspect = 0;
   let errors = 0;
+  let notMeasured = 0;
 
   for (const id of Object.keys(col.cells)) {
     const cell = col.cells[id];
@@ -62,6 +64,10 @@ export function gradeColumn(col, shipBar, critical) {
       continue; // excluded, blocks ship
     }
     const verdict = effective(cell);
+    if (verdict === "NOT-MEASURED") {
+      notMeasured++;
+      continue;
+    }
     if (verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS") {
       errors++;
       continue;
@@ -76,7 +82,7 @@ export function gradeColumn(col, shipBar, critical) {
   }
 
   if (col.partial === true) {
-    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors };
+    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured };
   }
 
   const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
@@ -89,9 +95,10 @@ export function gradeColumn(col, shipBar, critical) {
     (!shipBar.no_critical_fail || criticalFails === 0) &&
     bFails === 0 &&
     suspect === 0 &&
-    errors === 0;
+    errors === 0 &&
+    notMeasured === 0;
 
-  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors };
+  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured };
 }
 
 /**
@@ -107,7 +114,7 @@ export function gradeColumn(col, shipBar, critical) {
  */
 export function liftClass(liftCell, greenCell) {
   const conclusive = (verdict, suspect) =>
-    !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
 
   const redOk = conclusive(liftCell.red, liftCell.redSuspect);
   const greenOk = conclusive(effective(greenCell), !!greenCell.suspect && !greenCell.override);

package/assets/report.template.html

@@ -25,6 +25,7 @@
   .cell.PASS { background: rgba(31,139,76,.22); }
   .cell.FAIL { background: rgba(192,57,43,.24); }
   .cell.ERROR { background: rgba(107,114,128,.22); }
+  .cell.NOT-MEASURED { background: rgba(180,83,9,.18); }
   .cell.empty { color: var(--dim); cursor: default; font-weight: 400; }
   .cell .ov { display: block; font-size: 11px; color: var(--over); font-weight: 600; }
   .cell .ov.unsaved { color: #b45309; font-style: italic; }
@@ -49,7 +50,7 @@
      amber rather than red — missing evidence is not a failing gate. */
   .cell .lm.obj-pass { color: var(--pass); }
   .cell .lm.obj-fail { color: var(--fail); }
-  .cell .lm.obj-error { color: #b45309; }
+  .cell .lm.obj-error, .cell .lm.obj-not-measured { color: #b45309; }
   /* Adjudication. `unresolved` is amber for the same reason `suspect` is: it must
      never read as a clean PASS or FAIL. */
   .cell .lm.adj-settled { color: var(--dim); }
@@ -94,6 +95,7 @@
   .tgrid td.tc.PASS { color: #16a34a; }
   .tgrid td.tc.FAIL { color: #dc2626; }
   .tgrid td.tc.ERROR { color: #a855f7; }
+  .tgrid td.tc.NOT-MEASURED { color: #b45309; }
   .tgrid td.tc.suspect { color: #b45309; }
   .tgrid td.tc.absent { color: var(--dim, #bbb); }
   .dim { color: var(--dim, #888); font-weight: 400; }
@@ -429,7 +431,7 @@ function sparkline(runs) {
 function renderTrends(data) {
   const box = document.getElementById("trends");
   if (!data.models.length) { box.innerHTML = `<div class="empty-state">No runs yet.</div>`; return; }
-  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!" };
+  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", "NOT-MEASURED": "∅" };
   let html = "";
   for (const m of data.models) {
     const last = m.runs[m.runs.length - 1].grade;

package/dist/cli.d.ts

@@ -30,13 +30,15 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
  * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
  * than reconcile two numbers in prose.
  */
+export declare function cmdJudgeAgreement(args: Args): void;
 export declare function cmdMutationTest(): Promise<void>;
 /**
- * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
- * whose gate verdict flips from fail to pass, which the judge never saw and must now
- * be shown. Prints the cost before making those calls.
+ * Re-evaluate needle gates against the saved staged diffs — no subject call. A rep
+ * whose gate verdict flips from fail to pass was never judged and costs one judge call.
+ * The completion summary reports how many judge calls were made.
  */
 export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
+export declare function cmdScreen(args: Args): Promise<void>;
 export declare function cmdInit(args: Args): Promise<void>;
 export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 /**

package/dist/cli.js

@@ -3,10 +3,12 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync
 import { load as yamlLoad } from "js-yaml";
 import { basename, dirname, join, resolve, relative } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, runTrajectoryMutationSelfTest, resolveArm, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, judgeAgreement, assertNotDowngraded, downgradeWarning, isScoredMode, runTrajectoryMutationSelfTest, resolveArm, isFreeOfflineCommand, screenResults, formatScreen, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
+import { runSelfScreeningMutationCases } from "./mutation-catalogue.js";
 import { runCompareCommand } from "./compare.js";
+import { cmdQualification } from "./qualification.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 // The judge default lives in core (`defaultJudge()`), which resolves
 // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
@@ -388,9 +390,24 @@ export async function cmdGrade(args, adapterOverride) {
  * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
  * than reconcile two numbers in prose.
  */
+export function cmdJudgeAgreement(args) {
+    const raw = args._[0];
+    if (!raw)
+        throw new Error("usage: skill-harness judge-agreement <run-dir>");
+    const runDir = resolve(raw);
+    if (!existsSync(runDir))
+        throw new Error(`run dir not found: ${runDir}`);
+    const report = judgeAgreement(readResults(runDir));
+    for (const cell of report.cells) {
+        console.log(`  ${cell.status === "agree" ? "=" : cell.status === "disagree" ? "≠" : "?"} ${cell.scenario}: ${cell.status} — ${cell.detail}${cell.judges.length ? ` (${cell.judges.join(" vs ")})` : ""}`);
+    }
+    console.log(`\njudge agreement: ${report.agree} agree / ${report.disagree} disagree / ${report.error} error; ${report.rate === null ? "n/a" : `${(report.rate * 100).toFixed(1)}%`} across ${report.comparable} comparable scenario(s).`);
+    console.log("offline report only; no model or judge calls.");
+}
 export async function cmdMutationTest() {
-    const report = runTrajectoryMutationSelfTest();
-    console.log(`trajectory assertion mutation self-test: baseline ${report.baseline}`);
+    const trajectory = runTrajectoryMutationSelfTest();
+    const report = { baseline: trajectory.baseline, cases: [...trajectory.cases, ...await runSelfScreeningMutationCases()] };
+    console.log(`mutation self-test: baseline ${report.baseline}`);
     for (const test of report.cases) {
         console.log(`  ${test.detected ? "✓" : "✗"} ${test.id}: ${test.status} — ${test.detail}`);
     }
@@ -423,9 +440,9 @@ async function cmdRescore(args) {
     console.log(`\n${runDirs.length} run(s) re-scored, ${moved} verdict(s) moved.`);
 }
 /**
- * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
- * whose gate verdict flips from fail to pass, which the judge never saw and must now
- * be shown. Prints the cost before making those calls.
+ * Re-evaluate needle gates against the saved staged diffs — no subject call. A rep
+ * whose gate verdict flips from fail to pass was never judged and costs one judge call.
+ * The completion summary reports how many judge calls were made.
  */
 export async function cmdRegate(args, adapterOverride) {
     const runDirs = args._;
@@ -462,7 +479,7 @@ export async function cmdRegate(args, adapterOverride) {
         moved += changes.length;
         calls += judgeCalls;
     }
-    // The cost line matters: regate is advertised as free, and it is — except for the
+    // The cost line matters: regate makes no subject call, but fail→pass reps require a judge.
     // flipped reps, which it must not spend silently.
     console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
 }
@@ -517,6 +534,13 @@ async function cmdRestamp(args) {
     console.log(`\n${runs} run(s) examined: ${upgraded} upgraded, ${unprovable} left alone, ${unchanged} already current.` +
         `${partial > 0 ? ` (${partial} upgraded only in part.)` : ""} No models were called.`);
 }
+export async function cmdScreen(args) {
+    if (args._.length === 0)
+        throw new Error("usage: skill-harness screen <run-dir> [<run-dir> ...]");
+    const results = args._.map(runDir => readResults(resolve(runDir)));
+    console.log(formatScreen(screenResults(results)));
+    console.log(`\n${results.length} retained result(s) screened; 0 subject calls, 0 judge calls.`);
+}
 async function cmdStability(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0] ?? "all";
@@ -860,6 +884,11 @@ export async function cmdLint(args) {
  * prints none.
  */
 export function help() {
+    const free = (command) => {
+        if (!isFreeOfflineCommand(command))
+            throw new Error(`${command} is not a free/offline command`);
+        return "free, offline";
+    };
     return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
@@ -877,19 +906,26 @@ export function help() {
                      [--auto-rejudge] [--secondary-judge p:m] [--tie-break-judge p:m]
                        ask again about untrustworthy cells (ambiguous / contradictory / non-unanimous /
                        ship-deciding). OFF by default; prints the exact MAX extra call count first.
-  mutation-test                                  prove trajectory assertions turn red (free, offline)
-  rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
-  regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
-  restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (free, offline; one-time migration)
-  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (free, offline)
+  mutation-test                                  prove trajectory + results/delivery/screen gates turn red (${free("mutation-test")})
+  judge-agreement <run-dir>                      compare two distinct persisted judge votes per scenario (${free("judge-agreement")})
+  rescore <run-dir>...                          re-score saved reps vs current spec thresholds (${free("rescore")})
+  regate <run-dir>...  [--judge prov:model]     re-evaluate saved gates (no subject call; judges fail→pass reps)
+  restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (${free("restamp")}; one-time migration)
+  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (${free("stability")})
+  screen <run-dir>...                           retained control/treatment + criterion rates (${free("screen")})
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
-  init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
+  init   <skill>     --skills <root> [--force]     scaffold a commented template spec (${free("init")})
   suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
-  list   --skills <root>                        discovered skills + spec status
-  lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
-  coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (free, offline)
-  affected <skill>   --skills <root> [--base ref]   which scenarios a change could touch (free, offline)
+  list   --skills <root>                        discovered skills + spec status (${free("list")})
+  lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (${free("lint")}; CI gate; exits non-zero on findings)
+  coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (${free("coverage")})
+  affected <skill>   --skills <root> [--base ref]   which scenarios a change could touch (${free("affected")})
+  qualification <prepare|start|status|poll|validate|panel|cell|abort>  durable qualification-runner-v1 lifecycle + offline panel collapse
+                     prepare --spool DIR --config FILE --request FILE [--expected-config-sha256 HEX] (required in production)
+                     start|status|poll|abort --spool DIR --id ID  (abort also requires --reason ID)
+                     start may resume only with --continuation-authority-file <private-0600-prebound-one-use-capability>
+                     validate --spool DIR  (all calls use external, schema-validated arms; no automatic retry)
 
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
@@ -908,10 +944,12 @@ export async function main(argv) {
         case "compare": return cmdCompare(args);
         case "grade": return cmdGrade(args);
         case "mutation-test": return cmdMutationTest();
+        case "judge-agreement": return cmdJudgeAgreement(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
         case "restamp": return cmdRestamp(args);
         case "stability": return cmdStability(args);
+        case "screen": return cmdScreen(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
         case "init": return cmdInit(args);
@@ -920,6 +958,7 @@ export async function main(argv) {
         case "lint": return cmdLint(args);
         case "coverage": return cmdCoverage(args);
         case "affected": return cmdAffected(args);
+        case "qualification": return cmdQualification(args);
         case "version":
         case "--version":
         case "-v":

package/dist/mutation-catalogue.d.ts

@@ -0,0 +1,8 @@
+export interface PermanentMutationCase {
+    id: string;
+    detected: boolean;
+    status: "FAIL" | "ERROR";
+    detail: string;
+}
+/** Permanent non-trajectory mutation cases exercised by the free mutation-test command. */
+export declare function runSelfScreeningMutationCases(): Promise<PermanentMutationCase[]>;

package/dist/mutation-catalogue.js

@@ -0,0 +1,95 @@
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parseSpec, planAdjudication, runSkillModel, screenResults, validateResults, } from "@skill-harness/core";
+import { authenticatePromptObservation, bindPromptObservation, normalizePromptPayload, observeProviderPayload, promptCaptureIsTrusted, } from "@skill-harness/adapters";
+const h = "a".repeat(64);
+const deliveryAssertion = (status = "PASS") => ({ kind: "skill_delivered", status, detail: status });
+function validV3() {
+    return {
+        schema: 3, skill: "mutation", harness: "pi", model: "fake:m", judge: { provider: "fake", model: "j" },
+        timestamp: "t", label: null, mode: "force", effective_grade: { passed: 1, total: 1, pct: 100, letter: "A", ship: true, note: "" },
+        subject_invocations: [{ scenario_id: "A1", repetition: 0, prompt: { capture_version: "prompt-provenance-v1", request_index: 0, raw_sha256: h, normalized_sha256: h, normalization_rule: "cwd-line-v1", bytes: 1, contract_sha256: h, contract_bytes: 1, contract_occurrences: 1, mechanism: "append-system-prompt", status: "PASS" } }],
+        scenarios: [{ id: "A1", criterion_count: 1, judge_verdict: "PASS", judge_reason: "ok", suspect: false, override: null, note: "", objective: { status: "PASS", assertions: [deliveryAssertion()] }, rep_judgments: [{ repetition: 0, recorded_verdict: "PASS", objective: { status: "PASS", assertions: [deliveryAssertion()] }, judgments: [{ ordinal: 1, judge: { provider: "fake", model: "j" }, verdict: "PASS", reason: "ok", suspect: false, criteria: [{ index: 1, verdict: "PASS", reason: "ok" }] }] }] }],
+    };
+}
+function catches(id, mutate, detail) {
+    const result = structuredClone(validV3());
+    mutate(result);
+    let detected = false;
+    try {
+        validateResults(result);
+    }
+    catch {
+        detected = true;
+    }
+    return { id, detected, status: detected ? "FAIL" : "ERROR", detail };
+}
+function screenFile(id, verdicts) {
+    const result = validV3();
+    result.scenarios[0].id = id;
+    result.scenarios[0].reps = verdicts.length;
+    result.subject_invocations = verdicts.map((_, repetition) => ({ ...structuredClone(result.subject_invocations[0]), scenario_id: id, repetition }));
+    result.scenarios[0].rep_judgments = verdicts.map((verdict, repetition) => ({ repetition, recorded_verdict: verdict, objective: { status: "PASS", assertions: [deliveryAssertion()] }, judgments: [{ ordinal: 1, judge: { provider: "fake", model: "j" }, verdict, reason: verdict, suspect: false, criteria: [{ index: 1, verdict, reason: verdict }] }] }));
+    return result;
+}
+async function objectiveSuppressesJudge() {
+    const dir = mkdtempSync(join(tmpdir(), "skill-harness-mutation-delivery-"));
+    let judgeCalls = 0;
+    try {
+        mkdirSync(join(dir, "tests"), { recursive: true });
+        writeFileSync(join(dir, "SKILL.md"), "---\nname: mutation\ndescription: fixture\n---\n\n## Contract\nDo it.\n", "utf8");
+        const specPath = join(dir, "tests/specification.yaml");
+        writeFileSync(specPath, "skill: mutation\njudge_persona: strict\nship_bar: { total: 1, min_pass: 1, no_critical_fail: true }\ncritical: []\nscenarios:\n  - id: A1\n    title: delivery\n    turns: [hi]\n    checklist: [does it]\n", "utf8");
+        const spec = parseSpec(readFileSync(specPath, "utf8"), specPath);
+        const adapter = {
+            name: "pi", observesPrompts: true, available: async () => true,
+            run: async (req) => { req.onPromptObservation?.({ capture_version: "prompt-provenance-v1", request_index: 0, raw_sha256: h, normalized_sha256: h, normalization_rule: "cwd-line-v1", bytes: 1, contract_sha256: h, contract_bytes: 1, contract_occurrences: 0, mechanism: "append-system-prompt", status: "NOT-MEASURED" }); return ">>> USER:\nhi\n\n<<< ASSISTANT:\nanswer\n"; },
+            judge: async () => { judgeCalls++; return "VERDICT: PASS\nREASON: wrong"; },
+        };
+        const out = await runSkillModel({ spec, skillDir: dir, specPath, adapter, model: { provider: "fake", model: "m" }, modelToken: "fake:m", judge: { provider: "fake-judge", model: "j" }, mode: "force", timestamp: "2026-09-04T01:00:00.000Z" });
+        const detected = judgeCalls === 0 && out.results.scenarios[0].judge_verdict === "NOT-MEASURED" && out.results.effective_grade.total === 0;
+        return { id: "delivery-objective-suppresses-judge", detected, status: detected ? "FAIL" : "ERROR", detail: "undelivered subject must never reach the judge or efficacy denominator" };
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}
+/** Permanent non-trajectory mutation cases exercised by the free mutation-test command. */
+export async function runSelfScreeningMutationCases() {
+    const contract = "## Contract\nDo it\n";
+    const one = observeProviderPayload({ instructions: contract }, contract, "append-system-prompt", 0);
+    const forged = { ...one, contract_sha256: "f".repeat(64) };
+    const ceiling = screenFile("C", ["PASS", "PASS", "PASS", "PASS", "FAIL"]);
+    for (const observation of ceiling.subject_invocations)
+        observation.prompt = { ...observation.prompt, mechanism: "none", contract_occurrences: 0, status: "PASS" };
+    const undelivered = screenFile("U", ["PASS", "PASS"]);
+    undelivered.subject_invocations[1].prompt = { ...undelivered.subject_invocations[1].prompt, contract_occurrences: 0, status: "NOT-MEASURED" };
+    const suspect = screenFile("S", ["PASS"]);
+    suspect.scenarios[0].rep_judgments[0].judgments[0].suspect = true;
+    const cases = [
+        catches("schema-v3-missing-observation-rejected", r => { delete r.subject_invocations; }, "schema v3 must reject missing delivery evidence"),
+        catches("schema-v3-panel-divergence-rejected", r => { r.scenarios[0].rep_judgments[0].recorded_verdict = "FAIL"; }, "stored panel result must equal recomputed clean votes"),
+        catches("schema-v3-missing-criterion-rejected", r => { r.scenarios[0].rep_judgments[0].judgments[0].criteria.pop(); }, "criterion omissions must remain explicit"),
+        catches("schema-v3-per-repetition-objective-rejected", r => { delete r.scenarios[0].rep_judgments[0].objective; }, "aggregate objective cannot hide a repetition"),
+        catches("schema-v3-top-level-verdict-rejected", r => { const panel = r.scenarios[0].rep_judgments[0]; panel.judgments[0].verdict = "FAIL"; panel.judgments[0].criteria = panel.judgments[0].criteria.map(vote => ({ ...vote, verdict: "FAIL" })); panel.recorded_verdict = "FAIL"; }, "top-level verdict must match retained repetition evidence"),
+        catches("schema-v3-unsupported-pass-rejected", r => { r.scenarios[0].rep_judgments[0].judgments = []; }, "behavioral PASS requires a retained clean judgment"),
+        catches("schema-v3-adjudication-state-rejected", r => { const primary = r.scenarios[0].rep_judgments[0].judgments[0]; r.scenarios[0].adjudication = { repetition: 0, trigger: "ship_deciding", state: "unresolved", judgments: [primary, { ...structuredClone(primary), ordinal: 2 }] }; }, "adjudication state and verdict must match clean votes"),
+        { id: "delivery-zero-not-measured", detected: observeProviderPayload({ instructions: "none" }, contract, "append-system-prompt", 0).status === "NOT-MEASURED", status: "FAIL", detail: "zero delivery is not product failure" },
+        { id: "delivery-duplicate-not-measured", detected: observeProviderPayload({ instructions: contract + contract }, contract, "append-system-prompt", 0).status === "NOT-MEASURED", status: "FAIL", detail: "duplicate delivery is not valid measurement" },
+        { id: "delivery-unobservable-error", detected: observeProviderPayload({}, contract, "append-system-prompt", 0).status === "ERROR", status: "FAIL", detail: "unsupported payload is instrument error" },
+        await objectiveSuppressesJudge(),
+        { id: "delivery-error-suppresses-adjudication", detected: planAdjudication({ cells: [{ id: "A1", verdict: "ERROR", reason: "instrument", suspect: false, deliveryStatus: "ERROR" }], scenarios: [{ id: "A1", title: "A1", critical: false, mode: "inline", turns: ["x"], checklist: ["y"], workspace: "none", remote: false }], shipBar: { total: 1, min_pass: 1, no_critical_fail: true }, critical: [], tieBreakAvailable: true }).maxAdditionalCalls === 0, status: "FAIL", detail: "instrument delivery ERROR cannot authorize adjudication spend" },
+        { id: "observer-normalization-scope", detected: JSON.stringify(normalizePromptPayload({ instructions: "Keep /a\nCurrent working directory: /a\nKeep /a" }, "cwd-line-v1")) === JSON.stringify({ instructions: "Keep /a\nCurrent working directory:<normalized>\nKeep /a" }), status: "FAIL", detail: "normalization changes only the cwd line" },
+        { id: "observer-contract-binding", detected: bindPromptObservation(authenticatePromptObservation(forged, "key"), contract, "append-system-prompt", 0, "key").status === "ERROR", status: "FAIL", detail: "child contract identity cannot replace parent binding" },
+        { id: "observer-mac-tamper-rejected", detected: bindPromptObservation({ ...authenticatePromptObservation(one, "key"), mac: "0".repeat(64) }, contract, "append-system-prompt", 0, "key").status === "ERROR", status: "FAIL", detail: "complete child observation requires parent authentication" },
+        { id: "observer-extension-forgery-rejected", detected: !promptCaptureIsTrusted(1), status: "FAIL", detail: "same-process subject extensions cannot produce trusted provenance" },
+        { id: "observer-argv-wiring", detected: promptCaptureIsTrusted(0) && !promptCaptureIsTrusted(1), status: "FAIL", detail: "observer is enabled only on the authenticated extension-free path" },
+        { id: "screen-ceiling-boundary", detected: screenResults([ceiling]).scenarios[0].classification === "CEILING", status: "FAIL", detail: "four of five control passes is CEILING" },
+        { id: "screen-undelivered-filter", detected: (() => { const row = screenResults([undelivered]).scenarios[0]; return row.treatment.n === 1 && row.not_measured === 1; })(), status: "FAIL", detail: "undelivered reps have a bucket outside efficacy" },
+        { id: "screen-suspect-filter", detected: screenResults([suspect]).scenarios[0].control.n === 0, status: "FAIL", detail: "suspect-only panels do not enter rates" },
+        { id: "screen-unsupported-pass-filter", detected: (() => { const unsupported = screenFile("P", ["PASS"]); unsupported.scenarios[0].rep_judgments[0].judgments = []; return screenResults([unsupported]).scenarios[0].control.n === 0; })(), status: "FAIL", detail: "recorded PASS without a clean vote does not enter efficacy" },
+    ];
+    return cases.map(test => ({ ...test, status: test.detected ? "FAIL" : "ERROR" }));
+}
+//# sourceMappingURL=mutation-catalogue.js.map

package/dist/qualification.d.ts

@@ -0,0 +1,7 @@
+export interface QualificationCliArgs {
+    _: string[];
+    flags: Record<string, string | true>;
+}
+export declare function qualificationSupervisorRuntimeArgs(mode: "production" | "test", scriptPath: string): string[];
+export declare function consumeQualificationContinuationAuthority(path: string): string;
+export declare function cmdQualification(args: QualificationCliArgs): Promise<void>;

package/dist/qualification.js

@@ -0,0 +1,210 @@
+import { spawn } from "node:child_process";
+import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, realpathSync, unlinkSync } from "node:fs";
+import { setTimeout as sleep } from "node:timers/promises";
+import { abortQualificationInvocation, checkQualificationAuthentication, pollQualificationInvocation, prepareQualificationInvocation, qualificationCanonicalJson, recordQualificationCell, recordQualificationJudgePanel, qualificationInvocationStatus, readQualificationSpoolConfig, superviseQualificationInvocation, validateQualificationRunnerSpool, verifyQualificationExecutable, } from "@skill-harness/core";
+function flag(args, name) {
+    const value = args.flags[name];
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function required(args, name) {
+    const value = flag(args, name);
+    if (!value)
+        throw new Error(`qualification ${args._[0] ?? "command"} requires --${name} <value>`);
+    return value;
+}
+function integerFlag(args, name, fallback) {
+    const raw = flag(args, name);
+    if (raw === undefined)
+        return fallback;
+    const value = Number(raw);
+    if (!Number.isSafeInteger(value) || value < 0)
+        throw new Error(`qualification --${name} must be a non-negative integer`);
+    return value;
+}
+export function qualificationSupervisorRuntimeArgs(mode, scriptPath) {
+    // Production supervisors execute built JavaScript with no inherited Node
+    // loader/import/debug hooks. Source-mode tests need exactly the repository's
+    // tsx import and nothing from the parent process.execArgv.
+    return mode === "test" && scriptPath.endsWith(".ts") ? ["--import", "tsx"] : [];
+}
+export function consumeQualificationContinuationAuthority(path) {
+    let fd;
+    try {
+        fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+        const stat = fstatSync(fd);
+        if (!stat.isFile())
+            throw new Error("qualification continuation authority must be a regular non-symlink file");
+        if (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))
+            throw new Error("qualification continuation authority file must be mode 0600 and owned by the qualification user");
+        if (stat.size < 32 || stat.size > 4096)
+            throw new Error("qualification continuation authority must be 32..4096 bytes");
+        const authority = readFileSync(fd, "utf8");
+        const pathStat = lstatSync(path);
+        if (pathStat.isSymbolicLink() || pathStat.dev !== stat.dev || pathStat.ino !== stat.ino)
+            throw new Error("qualification continuation authority path changed while it was consumed");
+        unlinkSync(path);
+        if (!authority.trim())
+            throw new Error("qualification continuation authority must not be blank");
+        return authority;
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+function print(value) {
+    process.stdout.write(`${qualificationCanonicalJson(value)}\n`);
+}
+export async function cmdQualification(args) {
+    const operation = args._[0];
+    if (!operation)
+        throw new Error("usage: skill-harness qualification <prepare|start|status|poll|validate|panel|cell|abort> ...");
+    if (operation === "prepare") {
+        const invocation = prepareQualificationInvocation({
+            spool_dir: required(args, "spool"),
+            config_path: required(args, "config"),
+            request_path: required(args, "request"),
+            expected_configuration_sha256: flag(args, "expected-config-sha256"),
+        });
+        print({ invocation_id: invocation.invocation_id, phase: "prepared", consumed_calls: 0 });
+        return;
+    }
+    const spool = required(args, "spool");
+    if (operation === "validate") {
+        print(validateQualificationRunnerSpool(spool));
+        return;
+    }
+    if (operation === "panel") {
+        print(recordQualificationJudgePanel({
+            spool_dir: spool,
+            panel_id: required(args, "panel-id"),
+            member_invocation_ids: required(args, "members").split(",").filter(Boolean),
+        }));
+        return;
+    }
+    if (operation === "cell") {
+        print(recordQualificationCell({
+            spool_dir: spool,
+            cell_id: required(args, "cell-id"),
+        }));
+        return;
+    }
+    const id = required(args, "id");
+    if (operation === "status") {
+        print(qualificationInvocationStatus(spool, id));
+        return;
+    }
+    if (operation === "poll") {
+        print(await pollQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            wait_ms: integerFlag(args, "wait-ms", 0),
+            interval_ms: Math.max(5, integerFlag(args, "interval-ms", 100)),
+        }));
+        return;
+    }
+    if (operation === "abort") {
+        print(await abortQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            reason: required(args, "reason"),
+        }));
+        return;
+    }
+    if (operation === "__supervise") {
+        const fdRaw = flag(args, "authority-fd");
+        if (fdRaw !== "3")
+            throw new Error("qualification internal authority descriptor must be fd 3");
+        let authority;
+        try {
+            authority = JSON.parse(readFileSync(3, "utf8"));
+        }
+        catch {
+            throw new Error("qualification internal authority payload is invalid JSON");
+        }
+        finally {
+            closeSync(3);
+        }
+        if (!authority || typeof authority !== "object" || Array.isArray(authority) ||
+            Object.keys(authority).sort().join(",") !== "authentication_authority,continuation_authority") {
+            throw new Error("qualification internal authority payload is not closed");
+        }
+        const payload = authority;
+        if (typeof payload.authentication_authority !== "string" ||
+            (payload.continuation_authority !== null && typeof payload.continuation_authority !== "string")) {
+            throw new Error("qualification internal authority payload has invalid fields");
+        }
+        await superviseQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            child_env: process.env,
+            authentication_authority: payload.authentication_authority,
+            continuation_authority: payload.continuation_authority ?? undefined,
+        });
+        return;
+    }
+    if (operation === "start") {
+        const config = readQualificationSpoolConfig(spool);
+        const before = qualificationInvocationStatus(spool, id);
+        if (before.phase === "terminal" || ((before.phase === "running" || before.phase === "launch-claimed") && before.supervisor_alive === true)) {
+            print(before);
+            return;
+        }
+        const needsContinuation = before.attempt === 1 && before.supervisor_alive !== true;
+        if (needsContinuation && !flag(args, "continuation-authority-file")) {
+            throw new Error("qualification interrupted claimed invocation requires --continuation-authority-file");
+        }
+        if (config.mode === "production") {
+            const pinned = verifyQualificationExecutable(config.runner.executable);
+            const running = realpathSync(process.argv[1]);
+            if (running !== pinned.realpath) {
+                throw new Error(`qualification runner executable mismatch: configuration pins ${pinned.realpath}, running ${running}`);
+            }
+        }
+        const auth = await checkQualificationAuthentication({ spool_dir: spool, invocation_id: id });
+        if (flag(args, "continuation-authority") !== undefined)
+            throw new Error("qualification continuation authority must use --continuation-authority-file, never argv text");
+        const continuationFile = flag(args, "continuation-authority-file");
+        const continuation = continuationFile ? consumeQualificationContinuationAuthority(continuationFile) : undefined;
+        const childArgs = [
+            ...qualificationSupervisorRuntimeArgs(config.mode, process.argv[1]),
+            process.argv[1],
+            "qualification", "__supervise",
+            "--spool", spool,
+            "--id", id,
+            "--authority-fd", "3",
+        ];
+        const supervisor = spawn(process.execPath, childArgs, {
+            cwd: process.cwd(),
+            env: auth.child_env,
+            detached: true,
+            stdio: ["ignore", "ignore", "ignore", "pipe"],
+        });
+        await new Promise((resolve, reject) => {
+            supervisor.once("spawn", resolve);
+            supervisor.once("error", reject);
+        });
+        const authorityPipe = supervisor.stdio[3];
+        if (!authorityPipe || !("write" in authorityPipe))
+            throw new Error("qualification private authority pipe was not created");
+        authorityPipe.end(JSON.stringify({ authentication_authority: auth.launch_authority, continuation_authority: continuation ?? null }));
+        supervisor.unref();
+        const deadline = Date.now() + integerFlag(args, "ack-wait-ms", 1500);
+        const priorSupervisor = qualificationCanonicalJson(before.supervisor);
+        let status = qualificationInvocationStatus(spool, id);
+        const acknowledged = () => needsContinuation
+            ? status.phase === "terminal" || (status.supervisor_alive === true && qualificationCanonicalJson(status.supervisor) !== priorSupervisor)
+            : status.phase !== "prepared";
+        while (!acknowledged() && Date.now() < deadline) {
+            await sleep(10);
+            status = qualificationInvocationStatus(spool, id);
+        }
+        if (!acknowledged()) {
+            throw new Error(`qualification supervisor did not durably acknowledge ${id} before the acknowledgement deadline`);
+        }
+        print(status);
+        return;
+    }
+    throw new Error(`unknown qualification operation: ${operation}`);
+}
+//# sourceMappingURL=qualification.js.map

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, appendJudgeHistory, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -115,17 +115,23 @@ export async function serveReview(opts) {
                     res.writeHead(404).end("unknown scenario");
                     return;
                 }
+                const prev = results.scenarios.find((s) => s.id === body.scenarioId);
+                if (!prev) {
+                    res.writeHead(404).end("scenario not in this run");
+                    return;
+                }
+                const delivery = prev.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
+                if (results.schema === 3 && delivery?.status !== "PASS") {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: `scenario ${body.scenarioId} is ${delivery?.status ?? "ERROR"}: delivery-gated evidence cannot be re-judged` }));
+                    return;
+                }
                 const adapter = opts.adapter ?? getAdapter(results.harness);
                 if (!(await adapter.available())) {
                     res.writeHead(400, { "content-type": "application/json" });
                     res.end(JSON.stringify({ ok: false, error: `harness \`${results.harness}\` is not on PATH` }));
                     return;
                 }
-                const prev = results.scenarios.find((s) => s.id === body.scenarioId);
-                if (!prev) {
-                    res.writeHead(404).end("scenario not in this run");
-                    return;
-                }
                 const threshold = effectiveThreshold(prev, scenario);
                 try {
                     const rr = await regradeScenario({
@@ -133,12 +139,21 @@ export async function serveReview(opts) {
                         specDir: dirname(specPath), threshold, mode: results.mode,
                         expectedReps: prev.reps ?? 1,
                     });
-                    const merged = results.scenarios.map((s) => 
-                    // Same contract as `grade`, through the same choke point.
-                    s.id === body.scenarioId
-                        ? rebuildScenarioResult({ ...rr, metrics: mergeScenarioMetrics(s.metrics, rr.metrics) }, s, { objective: "carry", adjudication: "drop" })
-                        : s);
+                    const merged = results.scenarios.map((s) => {
+                        if (s.id !== body.scenarioId)
+                            return s;
+                        const judge_history = appendJudgeHistory(s, results.judge, {
+                            judge: results.judge,
+                            verdict: rr.judge_verdict,
+                            reason: rr.judge_reason,
+                            suspect: rr.suspect,
+                            criteria: rr.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0]?.criteria,
+                        });
+                        // Same contract as `grade`, through the same choke point.
+                        return rebuildScenarioResult({ ...rr, metrics: mergeScenarioMetrics(s.metrics, rr.metrics), rep_judgments: carryRepObjectives(rr.rep_judgments, s.rep_judgments), judge_history }, s, { objective: "carry", adjudication: "drop" });
+                    });
                     const written = writeResults(column.runDir, {
+                        schema: results.schema, subject_invocations: results.subject_invocations,
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
                         partial: results.partial,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.11.0",
+    "version": "0.12.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -41,10 +41,10 @@
         "node": ">=20"
     },
     "scripts": {
-        "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
+        "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.11.0",
-        "@skill-harness/adapters": "0.11.0"
+        "@skill-harness/core": "0.12.0",
+        "@skill-harness/adapters": "0.12.0"
     }
 }