@skill-harness/cli 0.11.0 → 0.13.0

package/README.md

@@ -24,6 +24,7 @@ npm i -g @skill-harness/cli
 | `skill-harness grade <run-dir> [--judge p:m]` | re-grade saved transcripts |
 | `skill-harness review <skill> --skills <root> [--port N]` | serve the interactive review UI |
 | `skill-harness add-test <skill> --skills <root> --id ID --title T ...` | scaffold a new scenario |
+| `skill-harness qualification prepare\|start\|status\|poll\|validate\|abort ...` | durable external-config qualification lifecycle |
 
 ## More
 

package/assets/report.grade.js

@@ -28,6 +28,7 @@ export function effective(cell) {
   // Mechanical evidence outranks a prose judge. Objective PASS deliberately
   // forces nothing — the checklist judge still decides the behavioral rubric.
   if (cell.objective && cell.objective.status === "ERROR") return "ERROR";
+  if (cell.objective && cell.objective.status === "NOT-MEASURED") return "NOT-MEASURED";
   if (cell.objective && cell.objective.status === "FAIL") return "FAIL";
   return cell.judge_verdict;
 }
@@ -53,6 +54,7 @@ export function gradeColumn(col, shipBar, critical) {
   let bFails = 0;
   let suspect = 0;
   let errors = 0;
+  let notMeasured = 0;
 
   for (const id of Object.keys(col.cells)) {
     const cell = col.cells[id];
@@ -62,6 +64,10 @@ export function gradeColumn(col, shipBar, critical) {
       continue; // excluded, blocks ship
     }
     const verdict = effective(cell);
+    if (verdict === "NOT-MEASURED") {
+      notMeasured++;
+      continue;
+    }
     if (verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS") {
       errors++;
       continue;
@@ -76,7 +82,7 @@ export function gradeColumn(col, shipBar, critical) {
   }
 
   if (col.partial === true) {
-    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors };
+    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors, notMeasured };
   }
 
   const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
@@ -89,9 +95,10 @@ export function gradeColumn(col, shipBar, critical) {
     (!shipBar.no_critical_fail || criticalFails === 0) &&
     bFails === 0 &&
     suspect === 0 &&
-    errors === 0;
+    errors === 0 &&
+    notMeasured === 0;
 
-  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors };
+  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors, notMeasured };
 }
 
 /**
@@ -107,7 +114,7 @@ export function gradeColumn(col, shipBar, critical) {
  */
 export function liftClass(liftCell, greenCell) {
   const conclusive = (verdict, suspect) =>
-    !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+    !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
 
   const redOk = conclusive(liftCell.red, liftCell.redSuspect);
   const greenOk = conclusive(effective(greenCell), !!greenCell.suspect && !greenCell.override);

package/assets/report.template.html

@@ -25,6 +25,7 @@
   .cell.PASS { background: rgba(31,139,76,.22); }
   .cell.FAIL { background: rgba(192,57,43,.24); }
   .cell.ERROR { background: rgba(107,114,128,.22); }
+  .cell.NOT-MEASURED { background: rgba(180,83,9,.18); }
   .cell.empty { color: var(--dim); cursor: default; font-weight: 400; }
   .cell .ov { display: block; font-size: 11px; color: var(--over); font-weight: 600; }
   .cell .ov.unsaved { color: #b45309; font-style: italic; }
@@ -49,7 +50,7 @@
      amber rather than red — missing evidence is not a failing gate. */
   .cell .lm.obj-pass { color: var(--pass); }
   .cell .lm.obj-fail { color: var(--fail); }
-  .cell .lm.obj-error { color: #b45309; }
+  .cell .lm.obj-error, .cell .lm.obj-not-measured { color: #b45309; }
   /* Adjudication. `unresolved` is amber for the same reason `suspect` is: it must
      never read as a clean PASS or FAIL. */
   .cell .lm.adj-settled { color: var(--dim); }
@@ -94,6 +95,7 @@
   .tgrid td.tc.PASS { color: #16a34a; }
   .tgrid td.tc.FAIL { color: #dc2626; }
   .tgrid td.tc.ERROR { color: #a855f7; }
+  .tgrid td.tc.NOT-MEASURED { color: #b45309; }
   .tgrid td.tc.suspect { color: #b45309; }
   .tgrid td.tc.absent { color: var(--dim, #bbb); }
   .dim { color: var(--dim, #888); font-weight: 400; }
@@ -429,7 +431,7 @@ function sparkline(runs) {
 function renderTrends(data) {
   const box = document.getElementById("trends");
   if (!data.models.length) { box.innerHTML = `<div class="empty-state">No runs yet.</div>`; return; }
-  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!" };
+  const glyph = { PASS: "✓", FAIL: "✗", ERROR: "!", "NOT-MEASURED": "∅" };
   let html = "";
   for (const m of data.models) {
     const last = m.runs[m.runs.length - 1].grade;

package/dist/archive-learning.d.ts

@@ -0,0 +1,3 @@
+import type { Args } from './cli.js';
+/** Explicit operator requests, not a scheduler or model route. The inert host cannot invoke this CLI. */
+export declare function cmdArchiveLearning(args: Args): void;

package/dist/archive-learning.js

@@ -0,0 +1,153 @@
+import { constants, openSync, readSync, fstatSync, closeSync } from 'node:fs';
+import { isAbsolute } from 'node:path';
+import { createArchiveAccess, openArchiveAccess, readFixedOrderFacts, createWeeklyInvestigation, openWeeklyInvestigation, createTrustLifecycle, openTrustLifecycle } from '@skill-harness/adapters';
+import { parseLearningRequest } from '@skill-harness/core';
+/** Explicit operator requests, not a scheduler or model route. The inert host cannot invoke this CLI. */
+export function cmdArchiveLearning(args) {
+    if (args._.length !== 1 || Object.keys(args.flags).sort().join() !== 'request,state' || typeof args.flags.state !== 'string' || !isAbsolute(args.flags.state) || typeof args.flags.request !== 'string')
+        throw Error('archive weekly|trust|access requires --state /absolute/directory --request file');
+    const fd = openSync(args.flags.request, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    let text;
+    try {
+        const stat = fstatSync(fd);
+        if (!stat.isFile() || stat.size > 65536)
+            throw Error('bounded regular learning request required');
+        const bytes = Buffer.alloc(65537);
+        let n = 0;
+        while (n < bytes.length) {
+            const k = readSync(fd, bytes, n, bytes.length - n, n);
+            if (!k)
+                break;
+            n += k;
+        }
+        if (n > 65536)
+            throw Error('learning request exceeds bound');
+        text = new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, n));
+    }
+    finally {
+        closeSync(fd);
+    }
+    const request = parseLearningRequest(text);
+    if (Object.keys(request).sort().join() !== 'authority,input,operation')
+        throw Error('closed learning request required');
+    const input = request.input, authority = request.authority, state = args.flags.state;
+    let result;
+    if (args._[0] === 'access') {
+        if (request.operation === 'create')
+            result = createArchiveAccess(state, input, authority).inspect();
+        else {
+            const access = openArchiveAccess(state);
+            switch (request.operation) {
+                case 'inspect':
+                    result = access.inspect();
+                    break;
+                case 'preview-consent':
+                    result = access.previewConsent(input);
+                    break;
+                case 'consent': {
+                    const p = input;
+                    result = access.consent(p.grant, authority, p.now);
+                    break;
+                }
+                case 'preview-revocation':
+                    result = access.previewRevocation(input);
+                    break;
+                case 'revoke': {
+                    const p = input;
+                    access.revoke(p.id, authority, p.now);
+                    result = { recorded: true };
+                    break;
+                }
+                case 'facts': {
+                    const p = input;
+                    result = readFixedOrderFacts(access.inspect().policy.archiveRoot, p.manifestId, { directory: state, purpose: p.purpose }, p.expected, p.now);
+                    break;
+                }
+                default: throw Error('unsupported access operation; raw read/export is not a CLI operation');
+            }
+        }
+    }
+    else if (args._[0] === 'weekly') {
+        if (request.operation === 'create') {
+            result = createWeeklyInvestigation(state, input).inspect();
+        }
+        else {
+            const job = openWeeklyInvestigation(state);
+            switch (request.operation) {
+                case 'inspect':
+                    result = job.inspect();
+                    break;
+                case 'run':
+                    result = job.run(request.input);
+                    break;
+                case 'approve':
+                    result = job.approve(authority);
+                    break;
+                case 'preview': {
+                    const p = request.input;
+                    result = job.preview(p.specPath, p.scenario);
+                    break;
+                }
+                case 'promote':
+                    result = job.promote(input, authority);
+                    break;
+                case 'freeze':
+                    result = job.freeze(input, authority);
+                    break;
+                case 'preview-edit':
+                    result = job.previewEdit(input);
+                    break;
+                case 'edit':
+                    result = job.edit(input, authority);
+                    break;
+                case 'evaluation':
+                    result = job.evaluation();
+                    break;
+                case 'preview-screen':
+                    result = job.previewScreen(input);
+                    break;
+                case 'screen':
+                    result = job.screen(input, authority);
+                    break;
+                default: throw Error('unsupported weekly operation');
+            }
+        }
+    }
+    else if (args._[0] === 'trust') {
+        if (request.operation === 'create') {
+            result = createTrustLifecycle(state, input, authority).inspect(0);
+        }
+        else {
+            const job = openTrustLifecycle(state);
+            switch (request.operation) {
+                case 'inspect':
+                    result = job.inspect(input);
+                    break;
+                case 'predict':
+                    job.predict(input);
+                    result = { recorded: true };
+                    break;
+                case 'correct': {
+                    const p = request.input;
+                    job.correct(p.prior, p.prediction, p.reason);
+                    result = { recorded: true };
+                    break;
+                }
+                case 'outcome':
+                    job.outcome(input, authority);
+                    result = { recorded: true };
+                    break;
+                case 'expose': {
+                    const p = request.input;
+                    result = job.expose(p.id, p.now);
+                    break;
+                }
+                default: throw Error('unsupported trust operation');
+            }
+        }
+    }
+    else
+        throw Error('unsupported learning family');
+    console.log(JSON.stringify(result, null, 2));
+}
+//# sourceMappingURL=archive-learning.js.map

package/dist/archive.d.ts

@@ -0,0 +1,3 @@
+import type { Args } from "./cli.js";
+/** Explicit file ingestion/metadata inspection only; never launches a worker or publishes content. */
+export declare function cmdArchive(args: Args): Promise<void>;

package/dist/archive.js

@@ -0,0 +1,52 @@
+import { ingestPolicySource, inspectPolicyCheckpoint, observeArchiveSource } from "@skill-harness/adapters";
+import { cmdArchiveLearning } from "./archive-learning.js";
+/** Explicit file ingestion/metadata inspection only; never launches a worker or publishes content. */
+export async function cmdArchive(args) {
+    const operation = args._[0];
+    if (operation === "weekly" || operation === "trust" || operation === "access") {
+        cmdArchiveLearning(args);
+        return;
+    }
+    if (operation === "watch") {
+        if (args._.length !== 1 || Object.keys(args.flags).some(key => !["policy", "source", "previous", "interval-ms", "max-polls"].includes(key)))
+            throw new Error("unsupported archive watch option");
+        const required = (key) => { const value = args.flags[key]; if (typeof value !== "string" || !value)
+            throw new Error(`archive watch requires --${key}`); return value; };
+        const signal = new AbortController(), abort = () => signal.abort();
+        process.on("SIGINT", abort);
+        process.on("SIGTERM", abort);
+        try {
+            const result = await observeArchiveSource({ policyPath: required("policy"), sourceId: required("source"),
+                previousCheckpointId: args.flags.previous === undefined ? undefined : required("previous"),
+                intervalMs: args.flags["interval-ms"] === undefined ? 1000 : Number(required("interval-ms")), maxPolls: Number(required("max-polls")), signal: signal.signal,
+                onObservation: observation => console.log(JSON.stringify({ checkpointId: observation.checkpointId, policySha256: observation.policySha256, syntax: observation.syntax })),
+            });
+            console.log(JSON.stringify(result));
+        }
+        finally {
+            process.removeListener("SIGINT", abort);
+            process.removeListener("SIGTERM", abort);
+        }
+        return;
+    }
+    if (args._.length !== 1 || !["ingest", "inspect"].includes(operation)) {
+        throw new Error("usage: archive ingest|inspect --policy <file> --source <id> [--previous <checkpoint> | --checkpoint <id>]");
+    }
+    const allowed = operation === "ingest" ? ["policy", "source", "previous"] : ["policy", "source", "checkpoint"];
+    for (const key of Object.keys(args.flags))
+        if (!allowed.includes(key))
+            throw new Error(`unsupported archive option: ${key}`);
+    const required = (key) => {
+        const value = args.flags[key];
+        if (typeof value !== "string" || !value)
+            throw new Error(`archive requires --${key}`);
+        return value;
+    };
+    const policy = required("policy"), source = required("source");
+    const result = operation === "ingest"
+        ? ingestPolicySource(policy, source, args.flags.previous === undefined ? undefined : required("previous"))
+        : inspectPolicyCheckpoint(policy, source, required("checkpoint"));
+    // Successful storage/reporting is not successful work: missing/partial fields are never promoted to acceptance.
+    console.log(JSON.stringify(result, null, 2));
+}
+//# sourceMappingURL=archive.js.map

package/dist/cli.d.ts

@@ -30,13 +30,14 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
  * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
  * than reconcile two numbers in prose.
  */
-export declare function cmdMutationTest(): Promise<void>;
+export declare function cmdJudgeAgreement(args: Args): void;
 /**
- * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
- * whose gate verdict flips from fail to pass, which the judge never saw and must now
- * be shown. Prints the cost before making those calls.
+ * Re-evaluate needle gates against the saved staged diffs — no subject call. A rep
+ * whose gate verdict flips from fail to pass was never judged and costs one judge call.
+ * The completion summary reports how many judge calls were made.
  */
 export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
+export declare function cmdScreen(args: Args): Promise<void>;
 export declare function cmdInit(args: Args): Promise<void>;
 export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 /**

package/dist/cli.js

@@ -3,10 +3,12 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync
 import { load as yamlLoad } from "js-yaml";
 import { basename, dirname, join, resolve, relative } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, runTrajectoryMutationSelfTest, resolveArm, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, judgeAgreement, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, screenResults, formatScreen, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 import { runCompareCommand } from "./compare.js";
+import { cmdQualification } from "./qualification.js";
+import { cmdArchive } from "./archive.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 // The judge default lives in core (`defaultJudge()`), which resolves
 // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
@@ -388,16 +390,19 @@ export async function cmdGrade(args, adapterOverride) {
  * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
  * than reconcile two numbers in prose.
  */
-export async function cmdMutationTest() {
-    const report = runTrajectoryMutationSelfTest();
-    console.log(`trajectory assertion mutation self-test: baseline ${report.baseline}`);
-    for (const test of report.cases) {
-        console.log(`  ${test.detected ? "✓" : "✗"} ${test.id}: ${test.status} — ${test.detail}`);
-    }
-    const missed = report.cases.filter((test) => !test.detected);
-    console.log(`\n${report.cases.length - missed.length}/${report.cases.length} mutations detected; no model or judge calls.`);
-    if (missed.length)
-        process.exitCode = 1;
+export function cmdJudgeAgreement(args) {
+    const raw = args._[0];
+    if (!raw)
+        throw new Error("usage: skill-harness judge-agreement <run-dir>");
+    const runDir = resolve(raw);
+    if (!existsSync(runDir))
+        throw new Error(`run dir not found: ${runDir}`);
+    const report = judgeAgreement(readResults(runDir));
+    for (const cell of report.cells) {
+        console.log(`  ${cell.status === "agree" ? "=" : cell.status === "disagree" ? "≠" : "?"} ${cell.scenario}: ${cell.status} — ${cell.detail}${cell.judges.length ? ` (${cell.judges.join(" vs ")})` : ""}`);
+    }
+    console.log(`\njudge agreement: ${report.agree} agree / ${report.disagree} disagree / ${report.error} error; ${report.rate === null ? "n/a" : `${(report.rate * 100).toFixed(1)}%`} across ${report.comparable} comparable scenario(s).`);
+    console.log("offline report only; no model or judge calls.");
 }
 async function cmdRescore(args) {
     const runDirs = args._;
@@ -423,9 +428,9 @@ async function cmdRescore(args) {
     console.log(`\n${runDirs.length} run(s) re-scored, ${moved} verdict(s) moved.`);
 }
 /**
- * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
- * whose gate verdict flips from fail to pass, which the judge never saw and must now
- * be shown. Prints the cost before making those calls.
+ * Re-evaluate needle gates against the saved staged diffs — no subject call. A rep
+ * whose gate verdict flips from fail to pass was never judged and costs one judge call.
+ * The completion summary reports how many judge calls were made.
  */
 export async function cmdRegate(args, adapterOverride) {
     const runDirs = args._;
@@ -462,7 +467,7 @@ export async function cmdRegate(args, adapterOverride) {
         moved += changes.length;
         calls += judgeCalls;
     }
-    // The cost line matters: regate is advertised as free, and it is — except for the
+    // The cost line matters: regate makes no subject call, but fail→pass reps require a judge.
     // flipped reps, which it must not spend silently.
     console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
 }
@@ -517,6 +522,13 @@ async function cmdRestamp(args) {
     console.log(`\n${runs} run(s) examined: ${upgraded} upgraded, ${unprovable} left alone, ${unchanged} already current.` +
         `${partial > 0 ? ` (${partial} upgraded only in part.)` : ""} No models were called.`);
 }
+export async function cmdScreen(args) {
+    if (args._.length === 0)
+        throw new Error("usage: skill-harness screen <run-dir> [<run-dir> ...]");
+    const results = args._.map(runDir => readResults(resolve(runDir)));
+    console.log(formatScreen(screenResults(results)));
+    console.log(`\n${results.length} retained result(s) screened; 0 subject calls, 0 judge calls.`);
+}
 async function cmdStability(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0] ?? "all";
@@ -860,6 +872,11 @@ export async function cmdLint(args) {
  * prints none.
  */
 export function help() {
+    const free = (command) => {
+        if (!isFreeOfflineCommand(command))
+            throw new Error(`${command} is not a free/offline command`);
+        return "free, offline";
+    };
     return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
@@ -877,19 +894,28 @@ export function help() {
                      [--auto-rejudge] [--secondary-judge p:m] [--tie-break-judge p:m]
                        ask again about untrustworthy cells (ambiguous / contradictory / non-unanimous /
                        ship-deciding). OFF by default; prints the exact MAX extra call count first.
-  mutation-test                                  prove trajectory assertions turn red (free, offline)
-  rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
-  regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
-  restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (free, offline; one-time migration)
-  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (free, offline)
+  archive ingest|inspect|watch --policy file --source id  explicit external ingestion/metadata (${free("archive")})
+  archive weekly|trust|access --state /private/dir --request file  durable learning/consent lifecycle (${free("archive")})
+                        watch requires --max-polls N; optional --interval-ms N and --previous checkpoint
+  judge-agreement <run-dir>                      compare two distinct persisted judge votes per scenario (${free("judge-agreement")})
+  rescore <run-dir>...                          re-score saved reps vs current spec thresholds (${free("rescore")})
+  regate <run-dir>...  [--judge prov:model]     re-evaluate saved gates (no subject call; judges fail→pass reps)
+  restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (${free("restamp")}; one-time migration)
+  stability <skill|all> --skills <root> [--window N] [--all]  run-over-run verdict flips per scenario (${free("stability")})
+  screen <run-dir>...                           retained control/treatment + criterion rates (${free("screen")})
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
-  init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
+  init   <skill>     --skills <root> [--force]     scaffold a commented template spec (${free("init")})
   suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
-  list   --skills <root>                        discovered skills + spec status
-  lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
-  coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (free, offline)
-  affected <skill>   --skills <root> [--base ref]   which scenarios a change could touch (free, offline)
+  list   --skills <root>                        discovered skills + spec status (${free("list")})
+  lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (${free("lint")}; CI gate; exits non-zero on findings)
+  coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (${free("coverage")})
+  affected <skill>   --skills <root> [--base ref]   which scenarios a change could touch (${free("affected")})
+  qualification <prepare|start|status|poll|validate|panel|cell|abort>  durable qualification-runner-v1 lifecycle + offline panel collapse
+                     prepare --spool DIR --config FILE --request FILE [--expected-config-sha256 HEX] (required in production)
+                     start|status|poll|abort --spool DIR --id ID  (abort also requires --reason ID)
+                     start may resume only with --continuation-authority-file <private-0600-prebound-one-use-capability>
+                     validate --spool DIR  (all calls use external, schema-validated arms; no automatic retry)
 
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
@@ -907,11 +933,13 @@ export async function main(argv) {
         case "run": return cmdRun(args);
         case "compare": return cmdCompare(args);
         case "grade": return cmdGrade(args);
-        case "mutation-test": return cmdMutationTest();
+        case "archive": return cmdArchive(args);
+        case "judge-agreement": return cmdJudgeAgreement(args);
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
         case "restamp": return cmdRestamp(args);
         case "stability": return cmdStability(args);
+        case "screen": return cmdScreen(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
         case "init": return cmdInit(args);
@@ -920,6 +948,7 @@ export async function main(argv) {
         case "lint": return cmdLint(args);
         case "coverage": return cmdCoverage(args);
         case "affected": return cmdAffected(args);
+        case "qualification": return cmdQualification(args);
         case "version":
         case "--version":
         case "-v":

package/dist/qualification.d.ts

@@ -0,0 +1,7 @@
+export interface QualificationCliArgs {
+    _: string[];
+    flags: Record<string, string | true>;
+}
+export declare function qualificationSupervisorRuntimeArgs(mode: "production" | "test", scriptPath: string): string[];
+export declare function consumeQualificationContinuationAuthority(path: string): string;
+export declare function cmdQualification(args: QualificationCliArgs): Promise<void>;

package/dist/qualification.js

@@ -0,0 +1,210 @@
+import { spawn } from "node:child_process";
+import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, realpathSync, unlinkSync } from "node:fs";
+import { setTimeout as sleep } from "node:timers/promises";
+import { abortQualificationInvocation, checkQualificationAuthentication, pollQualificationInvocation, prepareQualificationInvocation, qualificationCanonicalJson, recordQualificationCell, recordQualificationJudgePanel, qualificationInvocationStatus, readQualificationSpoolConfig, superviseQualificationInvocation, validateQualificationRunnerSpool, verifyQualificationExecutable, } from "@skill-harness/core";
+function flag(args, name) {
+    const value = args.flags[name];
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function required(args, name) {
+    const value = flag(args, name);
+    if (!value)
+        throw new Error(`qualification ${args._[0] ?? "command"} requires --${name} <value>`);
+    return value;
+}
+function integerFlag(args, name, fallback) {
+    const raw = flag(args, name);
+    if (raw === undefined)
+        return fallback;
+    const value = Number(raw);
+    if (!Number.isSafeInteger(value) || value < 0)
+        throw new Error(`qualification --${name} must be a non-negative integer`);
+    return value;
+}
+export function qualificationSupervisorRuntimeArgs(mode, scriptPath) {
+    // Production supervisors execute built JavaScript with no inherited Node
+    // loader/import/debug hooks. Source-mode tests need exactly the repository's
+    // tsx import and nothing from the parent process.execArgv.
+    return mode === "test" && scriptPath.endsWith(".ts") ? ["--import", "tsx"] : [];
+}
+export function consumeQualificationContinuationAuthority(path) {
+    let fd;
+    try {
+        fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+        const stat = fstatSync(fd);
+        if (!stat.isFile())
+            throw new Error("qualification continuation authority must be a regular non-symlink file");
+        if (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))
+            throw new Error("qualification continuation authority file must be mode 0600 and owned by the qualification user");
+        if (stat.size < 32 || stat.size > 4096)
+            throw new Error("qualification continuation authority must be 32..4096 bytes");
+        const authority = readFileSync(fd, "utf8");
+        const pathStat = lstatSync(path);
+        if (pathStat.isSymbolicLink() || pathStat.dev !== stat.dev || pathStat.ino !== stat.ino)
+            throw new Error("qualification continuation authority path changed while it was consumed");
+        unlinkSync(path);
+        if (!authority.trim())
+            throw new Error("qualification continuation authority must not be blank");
+        return authority;
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+function print(value) {
+    process.stdout.write(`${qualificationCanonicalJson(value)}\n`);
+}
+export async function cmdQualification(args) {
+    const operation = args._[0];
+    if (!operation)
+        throw new Error("usage: skill-harness qualification <prepare|start|status|poll|validate|panel|cell|abort> ...");
+    if (operation === "prepare") {
+        const invocation = prepareQualificationInvocation({
+            spool_dir: required(args, "spool"),
+            config_path: required(args, "config"),
+            request_path: required(args, "request"),
+            expected_configuration_sha256: flag(args, "expected-config-sha256"),
+        });
+        print({ invocation_id: invocation.invocation_id, phase: "prepared", consumed_calls: 0 });
+        return;
+    }
+    const spool = required(args, "spool");
+    if (operation === "validate") {
+        print(validateQualificationRunnerSpool(spool));
+        return;
+    }
+    if (operation === "panel") {
+        print(recordQualificationJudgePanel({
+            spool_dir: spool,
+            panel_id: required(args, "panel-id"),
+            member_invocation_ids: required(args, "members").split(",").filter(Boolean),
+        }));
+        return;
+    }
+    if (operation === "cell") {
+        print(recordQualificationCell({
+            spool_dir: spool,
+            cell_id: required(args, "cell-id"),
+        }));
+        return;
+    }
+    const id = required(args, "id");
+    if (operation === "status") {
+        print(qualificationInvocationStatus(spool, id));
+        return;
+    }
+    if (operation === "poll") {
+        print(await pollQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            wait_ms: integerFlag(args, "wait-ms", 0),
+            interval_ms: Math.max(5, integerFlag(args, "interval-ms", 100)),
+        }));
+        return;
+    }
+    if (operation === "abort") {
+        print(await abortQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            reason: required(args, "reason"),
+        }));
+        return;
+    }
+    if (operation === "__supervise") {
+        const fdRaw = flag(args, "authority-fd");
+        if (fdRaw !== "3")
+            throw new Error("qualification internal authority descriptor must be fd 3");
+        let authority;
+        try {
+            authority = JSON.parse(readFileSync(3, "utf8"));
+        }
+        catch {
+            throw new Error("qualification internal authority payload is invalid JSON");
+        }
+        finally {
+            closeSync(3);
+        }
+        if (!authority || typeof authority !== "object" || Array.isArray(authority) ||
+            Object.keys(authority).sort().join(",") !== "authentication_authority,continuation_authority") {
+            throw new Error("qualification internal authority payload is not closed");
+        }
+        const payload = authority;
+        if (typeof payload.authentication_authority !== "string" ||
+            (payload.continuation_authority !== null && typeof payload.continuation_authority !== "string")) {
+            throw new Error("qualification internal authority payload has invalid fields");
+        }
+        await superviseQualificationInvocation({
+            spool_dir: spool,
+            invocation_id: id,
+            child_env: process.env,
+            authentication_authority: payload.authentication_authority,
+            continuation_authority: payload.continuation_authority ?? undefined,
+        });
+        return;
+    }
+    if (operation === "start") {
+        const config = readQualificationSpoolConfig(spool);
+        const before = qualificationInvocationStatus(spool, id);
+        if (before.phase === "terminal" || ((before.phase === "running" || before.phase === "launch-claimed") && before.supervisor_alive === true)) {
+            print(before);
+            return;
+        }
+        const needsContinuation = before.attempt === 1 && before.supervisor_alive !== true;
+        if (needsContinuation && !flag(args, "continuation-authority-file")) {
+            throw new Error("qualification interrupted claimed invocation requires --continuation-authority-file");
+        }
+        if (config.mode === "production") {
+            const pinned = verifyQualificationExecutable(config.runner.executable);
+            const running = realpathSync(process.argv[1]);
+            if (running !== pinned.realpath) {
+                throw new Error(`qualification runner executable mismatch: configuration pins ${pinned.realpath}, running ${running}`);
+            }
+        }
+        const auth = await checkQualificationAuthentication({ spool_dir: spool, invocation_id: id });
+        if (flag(args, "continuation-authority") !== undefined)
+            throw new Error("qualification continuation authority must use --continuation-authority-file, never argv text");
+        const continuationFile = flag(args, "continuation-authority-file");
+        const continuation = continuationFile ? consumeQualificationContinuationAuthority(continuationFile) : undefined;
+        const childArgs = [
+            ...qualificationSupervisorRuntimeArgs(config.mode, process.argv[1]),
+            process.argv[1],
+            "qualification", "__supervise",
+            "--spool", spool,
+            "--id", id,
+            "--authority-fd", "3",
+        ];
+        const supervisor = spawn(process.execPath, childArgs, {
+            cwd: process.cwd(),
+            env: auth.child_env,
+            detached: true,
+            stdio: ["ignore", "ignore", "ignore", "pipe"],
+        });
+        await new Promise((resolve, reject) => {
+            supervisor.once("spawn", resolve);
+            supervisor.once("error", reject);
+        });
+        const authorityPipe = supervisor.stdio[3];
+        if (!authorityPipe || !("write" in authorityPipe))
+            throw new Error("qualification private authority pipe was not created");
+        authorityPipe.end(JSON.stringify({ authentication_authority: auth.launch_authority, continuation_authority: continuation ?? null }));
+        supervisor.unref();
+        const deadline = Date.now() + integerFlag(args, "ack-wait-ms", 1500);
+        const priorSupervisor = qualificationCanonicalJson(before.supervisor);
+        let status = qualificationInvocationStatus(spool, id);
+        const acknowledged = () => needsContinuation
+            ? status.phase === "terminal" || (status.supervisor_alive === true && qualificationCanonicalJson(status.supervisor) !== priorSupervisor)
+            : status.phase !== "prepared";
+        while (!acknowledged() && Date.now() < deadline) {
+            await sleep(10);
+            status = qualificationInvocationStatus(spool, id);
+        }
+        if (!acknowledged()) {
+            throw new Error(`qualification supervisor did not durably acknowledge ${id} before the acknowledgement deadline`);
+        }
+        print(status);
+        return;
+    }
+    throw new Error(`unknown qualification operation: ${operation}`);
+}
+//# sourceMappingURL=qualification.js.map

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, appendJudgeHistory, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -115,17 +115,23 @@ export async function serveReview(opts) {
                     res.writeHead(404).end("unknown scenario");
                     return;
                 }
+                const prev = results.scenarios.find((s) => s.id === body.scenarioId);
+                if (!prev) {
+                    res.writeHead(404).end("scenario not in this run");
+                    return;
+                }
+                const delivery = prev.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
+                if (results.schema === 3 && delivery?.status !== "PASS") {
+                    res.writeHead(400, { "content-type": "application/json" });
+                    res.end(JSON.stringify({ ok: false, error: `scenario ${body.scenarioId} is ${delivery?.status ?? "ERROR"}: delivery-gated evidence cannot be re-judged` }));
+                    return;
+                }
                 const adapter = opts.adapter ?? getAdapter(results.harness);
                 if (!(await adapter.available())) {
                     res.writeHead(400, { "content-type": "application/json" });
                     res.end(JSON.stringify({ ok: false, error: `harness \`${results.harness}\` is not on PATH` }));
                     return;
                 }
-                const prev = results.scenarios.find((s) => s.id === body.scenarioId);
-                if (!prev) {
-                    res.writeHead(404).end("scenario not in this run");
-                    return;
-                }
                 const threshold = effectiveThreshold(prev, scenario);
                 try {
                     const rr = await regradeScenario({
@@ -133,12 +139,21 @@ export async function serveReview(opts) {
                         specDir: dirname(specPath), threshold, mode: results.mode,
                         expectedReps: prev.reps ?? 1,
                     });
-                    const merged = results.scenarios.map((s) => 
-                    // Same contract as `grade`, through the same choke point.
-                    s.id === body.scenarioId
-                        ? rebuildScenarioResult({ ...rr, metrics: mergeScenarioMetrics(s.metrics, rr.metrics) }, s, { objective: "carry", adjudication: "drop" })
-                        : s);
+                    const merged = results.scenarios.map((s) => {
+                        if (s.id !== body.scenarioId)
+                            return s;
+                        const judge_history = appendJudgeHistory(s, results.judge, {
+                            judge: results.judge,
+                            verdict: rr.judge_verdict,
+                            reason: rr.judge_reason,
+                            suspect: rr.suspect,
+                            criteria: rr.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0]?.criteria,
+                        });
+                        // Same contract as `grade`, through the same choke point.
+                        return rebuildScenarioResult({ ...rr, metrics: mergeScenarioMetrics(s.metrics, rr.metrics), rep_judgments: carryRepObjectives(rr.rep_judgments, s.rep_judgments), judge_history }, s, { objective: "carry", adjudication: "drop" });
+                    });
                     const written = writeResults(column.runDir, {
+                        schema: results.schema, subject_invocations: results.subject_invocations,
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
                         timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
                         partial: results.partial,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.11.0",
+    "version": "0.13.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -41,10 +41,10 @@
         "node": ">=20"
     },
     "scripts": {
-        "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
+        "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.11.0",
-        "@skill-harness/adapters": "0.11.0"
+        "@skill-harness/core": "0.13.0",
+        "@skill-harness/adapters": "0.13.0"
     }
 }