@skill-harness/cli 0.8.0 → 0.9.0

package/assets/report.grade.js

@@ -24,7 +24,12 @@
 // a plain <script> in the browser (for the review UI).
 
 export function effective(cell) {
-  return cell.override || cell.judge_verdict;
+  if (cell.override) return cell.override;
+  // Mechanical evidence outranks a prose judge. Objective PASS deliberately
+  // forces nothing — the checklist judge still decides the behavioral rubric.
+  if (cell.objective && cell.objective.status === "ERROR") return "ERROR";
+  if (cell.objective && cell.objective.status === "FAIL") return "FAIL";
+  return cell.judge_verdict;
 }
 
 function letterFor(pct) {
@@ -47,6 +52,7 @@ export function gradeColumn(col, shipBar, critical) {
   let criticalFails = 0;
   let bFails = 0;
   let suspect = 0;
+  let errors = 0;
 
   for (const id of Object.keys(col.cells)) {
     const cell = col.cells[id];
@@ -55,8 +61,13 @@ export function gradeColumn(col, shipBar, critical) {
       suspect++;
       continue; // excluded, blocks ship
     }
+    const verdict = effective(cell);
+    if (verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS") {
+      errors++;
+      continue;
+    }
     total++;
-    if (effective(cell) === "PASS") {
+    if (verdict === "PASS") {
       passed++;
       continue;
     }
@@ -64,16 +75,23 @@ export function gradeColumn(col, shipBar, critical) {
     if (/^B/i.test(id)) bFails++;
   }
 
+  if (col.partial === true) {
+    return { passed: 0, total: 0, pct: 0, letter: "-", ship: false, criticalFails, bFails, suspect, errors };
+  }
+
   const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
   const letter = letterFor(pct);
+  const validBar = Number.isInteger(shipBar.total) && shipBar.total >= 1 && Number.isInteger(shipBar.min_pass) && shipBar.min_pass >= 1 && shipBar.min_pass <= shipBar.total;
   const ship =
+    validBar &&
     total >= shipBar.total &&
     passed >= shipBar.min_pass &&
     (!shipBar.no_critical_fail || criticalFails === 0) &&
     bFails === 0 &&
-    suspect === 0;
+    suspect === 0 &&
+    errors === 0;
 
-  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect };
+  return { passed, total, pct, letter, ship, criticalFails, bFails, suspect, errors };
 }
 
 /**

package/assets/report.template.html

@@ -165,13 +165,21 @@ function render() {
       // Green and force are scored; a red baseline column has no ship grade.
       gradeHtml = `<span class='badge no'>not scored (${escapeHtml(col.mode)})</span>`;
       if (g.suspect > 0) gradeHtml += ` — ${g.suspect} suspect`;
+      if (g.errors > 0) gradeHtml += ` — ${g.errors} infrastructure error`;
     } else {
       // g.ship is already false whenever g.suspect > 0 (gradeColumn's suspect gate), so
       // the NOT READY badge is forced automatically; we just surface the count here.
       const badge = g.ship ? "<span class='badge ship'>SHIP</span>" : "<span class='badge no'>NOT READY</span>";
       const suspectNote = g.suspect > 0 ? ` — ${g.suspect} suspect` : "";
-      gradeHtml = `${g.letter} (${g.pct}%) · ${g.passed}/${g.total}${suspectNote} ${badge}`;
+      const errorNote = g.errors > 0 ? ` — ${g.errors} infrastructure error${g.errors === 1 ? "" : "s"}` : "";
+      gradeHtml = `${g.letter} (${g.pct}%) · ${g.passed}/${g.total}${suspectNote}${errorNote} ${badge}`;
     }
+    const m = col.metrics;
+    const metricHtml = m && m.total_reps > 0
+      ? `<div class='grade' title='subject metrics reported by ${m.subject_metrics_reps}/${m.total_reps} reps'>` +
+        `${m.input_tokens == null ? "tokens n/a" : `${m.input_tokens}in/${m.output_tokens || 0}out`} · ` +
+        `${m.judge_calls} judge · ${m.wall_time_ms}ms · ${m.tool_calls == null ? "tools n/a" : `${m.tool_calls} tools`}</div>`
+      : "";
     // Lift answers "does this skill do anything?", which the grade alone cannot:
     // a column can score A because the model never needed the skill. Recomputed
     // from the live cells (see liftSummary) so overrides move it immediately.
@@ -189,7 +197,7 @@ function render() {
           : `lift 0 · no effect`;
       const inc = ls.inconclusive > 0 ? ` · ${ls.inconclusive} inconclusive` : "";
       const part = col.lift.partial ? " · partial" : "";
-      html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>` +
+      html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}` +
         `<div class='lift ${cls}' title='vs red baseline ${escapeHtml(col.lift.redTimestamp)} — ${ls.kept} passed without the skill too'>${body}${inc}${part}</div></th>`;
       return;
     }
@@ -199,7 +207,7 @@ function render() {
     if (none) {
       liftHtml = `<div class='lift none' title='${escapeHtml(none.title)}'>${escapeHtml(none.text)}</div>`;
     }
-    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${liftHtml}</th>`;
+    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}${liftHtml}</th>`;
   });
   html += "</tr></thead><tbody>";
   for (const scn of DATA.scenarios) {
@@ -281,6 +289,7 @@ async function openPanel(colIndex, scenarioId) {
     ${cell.stability ? `<div class="reason" style="color:#b45309"><b>⇄ boundary cell:</b> ${escapeHtml(cell.stability.note)}</div>` : ""}
     ${cell.objective ? `<div class="reason"><b>◉ objective ${escapeHtml(cell.objective.status)}:</b> ${escapeHtml(cell.objective.detail)}</div>` : ""}
     ${cell.adjudication ? `<div class="reason"${cell.adjudication.state === "unresolved" ? ' style="color:#b45309"' : ""}><b>⚖ adjudication ${escapeHtml(cell.adjudication.state)}</b> (${escapeHtml(cell.adjudication.trigger)}): ${escapeHtml(cell.adjudication.detail)}</div>` : ""}
+    ${cell.metrics ? `<div class="reason"><b>cost/latency:</b> ${cell.metrics.input_tokens == null ? "subject tokens unavailable" : `${cell.metrics.input_tokens} input / ${cell.metrics.output_tokens || 0} output / ${cell.metrics.cache_read_tokens || 0} cache-read`} (${cell.metrics.subject_metrics_reps}/${cell.metrics.total_reps} reps) · ${cell.metrics.judge_calls} judge + ${cell.metrics.judge_rejudge_calls} re-judge · ${cell.metrics.wall_time_ms}ms · ${cell.metrics.tool_calls == null ? "tool calls unavailable" : `${cell.metrics.tool_calls} tools, ${cell.metrics.delegated_children || 0} delegated, max concurrency ${cell.metrics.max_concurrency || 0}`}</div>` : ""}
     <div class="toggle">
       <button data-v="PASS" class="PASS ${cell.override === 'PASS' ? 'active PASS' : ''}">PASS</button>
       <button data-v="FAIL" class="FAIL ${cell.override === 'FAIL' ? 'active FAIL' : ''}">FAIL</button>

package/dist/cli.d.ts

@@ -13,8 +13,24 @@ export declare function parseRunTuning(args: Args): {
     reps: number;
     passThreshold: number;
 };
+export declare function releaseExitCode(summaries: Array<{
+    results: {
+        mode: string;
+        partial?: boolean;
+        effective_grade: {
+            ship: boolean;
+        };
+    };
+}>): 0 | 1;
 export declare function cmdRun(args: Args): Promise<void>;
+export declare function cmdCompare(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
+/**
+ * Re-score saved runs against the current spec's thresholds — no model or judge calls.
+ * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
+ * than reconcile two numbers in prose.
+ */
+export declare function cmdMutationTest(): Promise<void>;
 /**
  * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
  * whose gate verdict flips from fail to pass, which the judge never saw and must now

package/dist/cli.js

@@ -3,9 +3,10 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync
 import { load as yamlLoad } from "js-yaml";
 import { basename, dirname, join, resolve, relative } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, runTrajectoryMutationSelfTest, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
+import { runCompareCommand } from "./compare.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 // The judge default lives in core (`defaultJudge()`), which resolves
 // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
@@ -115,6 +116,9 @@ async function cmdList(args) {
     }
     console.log(`\n● = testable · ○ = no spec yet · ✗ = spec present but invalid`);
 }
+export function releaseExitCode(summaries) {
+    return summaries.some(({ results }) => isScoredMode(results.mode) && !results.partial && !results.effective_grade.ship) ? 1 : 0;
+}
 export async function cmdRun(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0];
@@ -212,6 +216,99 @@ export async function cmdRun(args) {
         }
     }
     console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
+    // A full delivered run is a release gate. NOT READY — including one critical
+    // failure hidden by a high aggregate — must be machine-visible to CI. Red
+    // baselines and partial/affected branch feedback are deliberately excluded.
+    if (releaseExitCode(summaries) !== 0)
+        process.exitCode = 1;
+}
+export async function cmdCompare(args, adapterOverride) {
+    const target = args._[0];
+    const reference = flagStr(args, "reference");
+    const candidateRoot = flagStr(args, "candidate");
+    if (!target || !reference || !candidateRoot) {
+        throw new Error("usage: skill-harness compare <skill|all> --reference <git-ref-or-skills-root> --candidate <skills-root> --model <provider:model> --reps N");
+    }
+    const judgeFlag = flagStr(args, "judge");
+    const judgeToken = judgeFlag ?? defaultJudge();
+    const judge = parseModelRef(judgeToken);
+    assertJudgeAllowed(judge, {
+        source: judgeFlag ? "--judge" : "the default judge (SKILL_HARNESS_JUDGE or the baked value)",
+        allowMetered: flagBool(args, "allow-metered-judge"),
+    });
+    const harnessName = flagStr(args, "harness", "pi");
+    const adapter = adapterOverride ?? getAdapter(harnessName);
+    if (!(await adapter.available()))
+        throw new Error(`harness \`${harnessName}\` is not on PATH`);
+    const mode = (flagStr(args, "mode", "force") || "force");
+    if (mode === "red")
+        throw new Error("compare measures a skill candidate, so --mode must be green or force (red is a no-skill baseline)");
+    const tuning = parseRunTuning(args);
+    const onlyRaw = flagStr(args, "only");
+    let only = onlyRaw ? onlyRaw.split(",").map((id) => id.trim()).filter(Boolean) : undefined;
+    const affected = flagBool(args, "affected");
+    if (affected && only)
+        throw new Error("--affected and --only both choose the comparison scenario set — pass one, not both");
+    if (affected) {
+        if (target === "all")
+            throw new Error("compare --affected currently requires one skill so each selected ID has an unambiguous spec");
+        const skill = resolveSkill(candidateRoot, target);
+        const spec = loadSpec(skill.specPath);
+        const selectionArgs = {
+            ...args,
+            flags: {
+                ...args.flags,
+                // A git-ref reference is the natural diff base. A reference directory
+                // has no common history, so the caller must supply --base explicitly.
+                ...(!flagStr(args, "base") && !existsSync(reference) ? { base: reference } : {}),
+            },
+        };
+        if (existsSync(reference) && !flagStr(args, "base")) {
+            throw new Error("compare --affected with a reference directory needs --base <git-ref> for candidate change selection");
+        }
+        const selected = await computeAffected(selectionArgs, spec.scenarios, skill.specPath);
+        console.log(formatAffected(selected, spec.scenarios.length));
+        only = selected.selected.map((entry) => entry.id);
+        if (only.length === 0) {
+            console.log("compare: no affected scenarios — 0 subject calls, 0 judge calls; no release claim produced");
+            return;
+        }
+    }
+    const threshold = (name) => {
+        const raw = flagStr(args, name);
+        if (raw === undefined)
+            return undefined;
+        const value = Number(raw);
+        if (!Number.isFinite(value) || value < 0)
+            throw new Error(`--${name} must be a non-negative ratio (got \`${raw}\`)`);
+        return value;
+    };
+    const thresholds = {
+        max_subject_token_increase: threshold("max-subject-token-increase"),
+        max_wall_time_increase: threshold("max-wall-time-increase"),
+        max_tool_call_increase: threshold("max-tool-call-increase"),
+    };
+    const hasThreshold = Object.values(thresholds).some((value) => value !== undefined);
+    const result = await runCompareCommand({
+        target,
+        reference,
+        candidateRoot,
+        models: resolveModels(args),
+        judgeToken,
+        mode,
+        reps: tuning.reps,
+        passThreshold: tuning.passThreshold,
+        parallel: Math.max(1, Number(flagStr(args, "parallel", "1")) || 1),
+        only,
+        canary: flagBool(args, "canary"),
+        output: flagStr(args, "output") || undefined,
+        thresholds: hasThreshold ? thresholds : undefined,
+        adapter,
+        now: nowIso,
+    });
+    console.log(`\ncomparison artifacts: ${result.outputDir}`);
+    if (result.exitCode !== 0)
+        process.exitCode = result.exitCode;
 }
 export async function cmdGrade(args, adapterOverride) {
     const runDir = args._[0];
@@ -283,6 +380,17 @@ export async function cmdGrade(args, adapterOverride) {
  * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
  * than reconcile two numbers in prose.
  */
+export async function cmdMutationTest() {
+    const report = runTrajectoryMutationSelfTest();
+    console.log(`trajectory assertion mutation self-test: baseline ${report.baseline}`);
+    for (const test of report.cases) {
+        console.log(`  ${test.detected ? "✓" : "✗"} ${test.id}: ${test.status} — ${test.detail}`);
+    }
+    const missed = report.cases.filter((test) => !test.detected);
+    console.log(`\n${report.cases.length - missed.length}/${report.cases.length} mutations detected; no model or judge calls.`);
+    if (missed.length)
+        process.exitCode = 1;
+}
 async function cmdRescore(args) {
     const runDirs = args._;
     if (runDirs.length === 0)
@@ -750,10 +858,15 @@ export function help() {
                      [--affected --base <git-ref>]  run only the scenarios a change could touch (partial; never SHIPs)
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
                      [--canary]  green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
+  compare <skill|all> --reference <git-ref-or-skills-root> --candidate <skills-root>
+                     [--model prov:model ...] [--mode green|force] [--judge prov:model] [--reps N] [--only IDs | --affected --base ref]
+                     [--output dir] [--max-subject-token-increase R] [--max-wall-time-increase R] [--max-tool-call-increase R]
+                       paired setup (not seeded sampling); critical regression exit 2, ordinary regression exit 1
   grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
                      [--auto-rejudge] [--secondary-judge p:m] [--tie-break-judge p:m]
                        ask again about untrustworthy cells (ambiguous / contradictory / non-unanimous /
                        ship-deciding). OFF by default; prints the exact MAX extra call count first.
+  mutation-test                                  prove trajectory assertions turn red (free, offline)
   rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   regate <run-dir>...  [--judge prov:model]     re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
   restamp <skill|all> --skills <root> [--from <git-ref>]   record the model-visible skill digest on runs that still match (free, offline; one-time migration)
@@ -781,7 +894,9 @@ export async function main(argv) {
     const args = parseArgs(argv.slice(1));
     switch (cmd) {
         case "run": return cmdRun(args);
+        case "compare": return cmdCompare(args);
         case "grade": return cmdGrade(args);
+        case "mutation-test": return cmdMutationTest();
         case "rescore": return cmdRescore(args);
         case "regate": return cmdRegate(args);
         case "restamp": return cmdRestamp(args);

package/dist/compare.d.ts

@@ -0,0 +1,26 @@
+import { buildComparison, type ComparisonThresholds, type HarnessAdapter, type RunMode } from "@skill-harness/core";
+export interface CompareCommandOptions {
+    target: string;
+    reference: string;
+    candidateRoot: string;
+    models: string[];
+    judgeToken: string;
+    mode: RunMode;
+    reps: number;
+    passThreshold: number;
+    parallel: number;
+    only?: string[];
+    canary?: boolean;
+    output?: string;
+    thresholds?: ComparisonThresholds;
+    adapter: HarnessAdapter;
+    now: () => string;
+    log?: (line: string) => void;
+}
+export interface CompareCommandResult {
+    outputDir: string;
+    reports: ReturnType<typeof buildComparison>[];
+    exitCode: 0 | 1 | 2;
+}
+/** Execute the same immutable plan against reference and candidate snapshots. */
+export declare function runCompareCommand(options: CompareCommandOptions): Promise<CompareCommandResult>;

package/dist/compare.js

@@ -0,0 +1,232 @@
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { execFileSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+import { dump as yamlDump } from "js-yaml";
+import { buildComparison, comparisonExitCode, discover, fileSha256, FIXTURE_PREFIX, formatComparison, HARNESS_VERSION, loadSpec, modelSlug, parseModelRef, resolveSkill, runSkillModel, SKILL_KEY, SKILL_PROMPT_KEY, sourceHashes, } from "@skill-harness/core";
+/** Execute the same immutable plan against reference and candidate snapshots. */
+export async function runCompareCommand(options) {
+    const log = options.log ?? console.log;
+    const temp = mkdtempSync(join(tmpdir(), "skill-harness-compare-"));
+    try {
+        const candidate = snapshotExisting(options.candidateRoot, join(temp, "candidate"));
+        const reference = existsSync(options.reference)
+            ? snapshotExisting(options.reference, join(temp, "reference"))
+            : snapshotGitRef(options.candidateRoot, options.reference, join(temp, "reference"));
+        const candidateSkills = options.target === "all"
+            ? discover(candidate.skillsRoot).filter((skill) => skill.hasSpec)
+            : [resolveSkill(candidate.skillsRoot, options.target)];
+        const referenceSkills = new Map((options.target === "all"
+            ? discover(reference.skillsRoot).filter((skill) => skill.hasSpec)
+            : [resolveSkill(reference.skillsRoot, options.target)])
+            .map((skill) => [skill.name, skill]));
+        if (!candidateSkills.length)
+            throw new Error(`no candidate skills with specs under ${options.candidateRoot}`);
+        const outputDir = resolve(options.output ?? join(options.candidateRoot, ".skill-harness", "comparisons", timestampSlug(options.now())));
+        mkdirSync(outputDir, { recursive: true });
+        const reports = [];
+        const fixtures = (sources) => Object.fromEntries(Object.entries(sources).filter(([key]) => key.startsWith(FIXTURE_PREFIX)));
+        const inputs = (sources) => Object.fromEntries(Object.entries(sources).filter(([key]) => key !== SKILL_KEY && key !== SKILL_PROMPT_KEY));
+        // Materialize and validate the ENTIRE skill × model plan before the first
+        // paid call. `compare all` must not spend on skill 1 and only then discover
+        // that skill 7 is not a valid pair.
+        const plans = candidateSkills.map((candidateSkill) => {
+            const referenceSkill = referenceSkills.get(candidateSkill.name);
+            if (!referenceSkill)
+                throw new Error(`reference has no matching skill \`${candidateSkill.name}\``);
+            const candidateSpec = loadSpec(candidateSkill.specPath);
+            const referenceSpec = loadSpec(referenceSkill.specPath);
+            const candidateIds = candidateSpec.scenarios.map((scenario) => scenario.id).sort();
+            const referenceIds = referenceSpec.scenarios.map((scenario) => scenario.id).sort();
+            if (JSON.stringify(candidateIds) !== JSON.stringify(referenceIds))
+                throw new Error(`${candidateSkill.name}: reference/candidate scenario IDs differ`);
+            const refSpecDigest = fileSha256(referenceSkill.specPath);
+            const candSpecDigest = fileSha256(candidateSkill.specPath);
+            if (!refSpecDigest || !candSpecDigest || refSpecDigest !== candSpecDigest) {
+                throw new Error(`${candidateSkill.name}: reference/candidate specification.yaml differs — paired comparison changes only the skill under test`);
+            }
+            const refSources = sourceHashes({ skillDir: referenceSkill.dir, specDir: dirname(referenceSkill.specPath), scenarios: referenceSpec.scenarios, judgePersona: referenceSpec.judge_persona });
+            const candSources = sourceHashes({ skillDir: candidateSkill.dir, specDir: dirname(candidateSkill.specPath), scenarios: candidateSpec.scenarios, judgePersona: candidateSpec.judge_persona });
+            if (JSON.stringify(Object.entries(inputs(refSources)).sort()) !== JSON.stringify(Object.entries(inputs(candSources)).sort())) {
+                throw new Error(`${candidateSkill.name}: reference/candidate fixtures or other test inputs differ — refusing before model calls`);
+            }
+            return { candidateSkill, referenceSkill, candidateSpec, referenceSpec, refSpecDigest, candSpecDigest, refSources, candSources };
+        });
+        const modelPlans = options.models.map((token) => ({ token, model: parseModelRef(token) }));
+        const judge = parseModelRef(options.judgeToken);
+        for (const { candidateSkill, referenceSkill, candidateSpec, referenceSpec, refSpecDigest, candSpecDigest, refSources, candSources } of plans) {
+            for (const { token: modelToken, model } of modelPlans) {
+                log(`\n▶ compare ${candidateSkill.name} · ${modelToken} · reference then candidate`);
+                const timestamp = options.now();
+                const referenceRun = await runSkillModel({
+                    spec: referenceSpec,
+                    skillDir: referenceSkill.dir,
+                    specPath: referenceSkill.specPath,
+                    adapter: options.adapter,
+                    model,
+                    modelToken,
+                    judge,
+                    mode: options.mode,
+                    timestamp,
+                    label: "compare-reference",
+                    concurrency: options.parallel,
+                    reps: options.reps,
+                    passThreshold: options.passThreshold,
+                    only: options.only,
+                    canary: options.canary,
+                    onProgress: (line) => log(`  reference ${line}`),
+                });
+                const candidateRun = await runSkillModel({
+                    spec: candidateSpec,
+                    skillDir: candidateSkill.dir,
+                    specPath: candidateSkill.specPath,
+                    adapter: options.adapter,
+                    model,
+                    modelToken,
+                    judge,
+                    mode: options.mode,
+                    timestamp,
+                    label: "compare-candidate",
+                    concurrency: options.parallel,
+                    reps: options.reps,
+                    passThreshold: options.passThreshold,
+                    only: options.only,
+                    canary: options.canary,
+                    onProgress: (line) => log(`  candidate ${line}`),
+                });
+                const digests = {
+                    reference: {
+                        skill: mustDigest(join(referenceSkill.dir, "SKILL.md")),
+                        spec: refSpecDigest,
+                        fixtures: fixtures(refSources),
+                        inputs: inputs(refSources),
+                    },
+                    candidate: {
+                        skill: mustDigest(join(candidateSkill.dir, "SKILL.md")),
+                        spec: candSpecDigest,
+                        fixtures: fixtures(candSources),
+                        inputs: inputs(candSources),
+                    },
+                    harness: runtimeHarnessDigest(),
+                    model: sha256(modelToken),
+                    judge: sha256(options.judgeToken),
+                    environment: {
+                        node: process.version,
+                        platform: process.platform,
+                        arch: process.arch,
+                        harness_version: HARNESS_VERSION,
+                        harness_cli_reference: referenceRun.results.harness_cli_version ?? "unavailable",
+                        harness_cli_candidate: candidateRun.results.harness_cli_version ?? "unavailable",
+                    },
+                };
+                const report = buildComparison({
+                    skill: candidateSpec.skill,
+                    model: modelToken,
+                    mode: options.mode,
+                    reps: options.reps,
+                    judge: options.judgeToken,
+                    reference: referenceRun.results,
+                    candidate: candidateRun.results,
+                    critical: candidateSpec.critical,
+                    digests,
+                    partial: Boolean(options.only?.length),
+                    thresholds: options.thresholds,
+                });
+                reports.push(report);
+                const cellDir = join(outputDir, candidateSkill.name, modelSlug(model));
+                mkdirSync(cellDir, { recursive: true });
+                cpSync(referenceRun.runDir, join(cellDir, "reference"), { recursive: true });
+                cpSync(candidateRun.runDir, join(cellDir, "candidate"), { recursive: true });
+                writeFileSync(join(cellDir, "comparison.yaml"), yamlDump(report, { lineWidth: 120 }), "utf8");
+                writeFileSync(join(cellDir, "comparison.txt"), `${formatComparison(report)}\n`, "utf8");
+                log(`\n${formatComparison(report)}\n  artifacts: ${cellDir}`);
+            }
+        }
+        const exitCode = reports.reduce((highest, report) => Math.max(highest, comparisonExitCode(report)), 0);
+        return { outputDir, reports, exitCode };
+    }
+    finally {
+        rmSync(temp, { recursive: true, force: true });
+    }
+}
+function snapshotExisting(skillsRoot, destination) {
+    const absolute = realpathSync(resolve(skillsRoot));
+    const repo = gitRoot(absolute);
+    const sourceRoot = repo ? realpathSync(repo) : absolute;
+    const inside = relativeInside(sourceRoot, absolute);
+    copyTree(sourceRoot, destination);
+    return { skillsRoot: join(destination, inside) };
+}
+function snapshotGitRef(candidateSkillsRoot, ref, destination) {
+    const absolute = realpathSync(resolve(candidateSkillsRoot));
+    const rawRepo = gitRoot(absolute);
+    const repo = rawRepo ? realpathSync(rawRepo) : null;
+    if (!repo)
+        throw new Error(`--reference ${ref} is not a directory and --candidate is not inside a git repository`);
+    execFileSync("git", ["clone", "--quiet", "--no-checkout", repo, destination], { stdio: ["ignore", "ignore", "pipe"] });
+    try {
+        execFileSync("git", ["checkout", "--quiet", "--detach", ref], { cwd: destination, stdio: ["ignore", "ignore", "pipe"] });
+    }
+    catch (error) {
+        throw new Error(`cannot materialize reference git ref \`${ref}\`: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return { skillsRoot: join(destination, relativeInside(repo, absolute)) };
+}
+function relativeInside(root, target) {
+    const rel = relative(root, target);
+    if (rel === ".." || rel.startsWith(`..${sep}`) || resolve(root, rel) !== target) {
+        throw new Error(`skills root ${target} is outside snapshot root ${root}`);
+    }
+    return rel;
+}
+function gitRoot(path) {
+    try {
+        return execFileSync("git", ["-C", path, "rev-parse", "--show-toplevel"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
+    }
+    catch {
+        return null;
+    }
+}
+function copyTree(source, destination) {
+    cpSync(source, destination, {
+        recursive: true,
+        filter: (path) => {
+            const rel = relative(source, path).replace(/\\/g, "/");
+            if (!rel)
+                return true;
+            return !rel.split("/").some((part) => part === ".git" || part === "node_modules" || part === ".skill-harness") && !/(^|\/)tests\/results(\/|$)/.test(rel);
+        },
+    });
+}
+function mustDigest(path) { const digest = fileSha256(path); if (!digest)
+    throw new Error(`cannot digest ${path}`); return digest; }
+function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
+function timestampSlug(value) { return value.replace(/[:.]/g, "-"); }
+function runtimeHarnessDigest() {
+    const hash = createHash("sha256").update(`skill-harness/${HARNESS_VERSION}\0`);
+    const require = createRequire(import.meta.url);
+    const cliDir = dirname(fileURLToPath(import.meta.url));
+    const roots = [
+        cliDir,
+        join(dirname(cliDir), "assets"),
+        dirname(require.resolve("@skill-harness/core")),
+        dirname(require.resolve("@skill-harness/adapters")),
+    ].filter((root, index, all) => existsSync(root) && all.indexOf(root) === index);
+    if (!roots.length)
+        throw new Error("cannot locate running harness artifacts for comparison digest");
+    roots.forEach((root, rootIndex) => {
+        for (const file of filesUnder(root).sort()) {
+            hash.update(`${rootIndex}/${relative(root, file).replace(/\\/g, "/")}`).update("\0").update(readFileSync(file)).update("\0");
+        }
+    });
+    return hash.digest("hex");
+}
+function filesUnder(path) {
+    if (statSync(path).isFile())
+        return [path];
+    return readdirSync(path, { withFileTypes: true }).flatMap((entry) => entry.isDirectory() ? filesUnder(join(path, entry.name)) : entry.isFile() ? [join(path, entry.name)] : []);
+}
+//# sourceMappingURL=compare.js.map

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, rebuildScenarioResult, mergeScenarioMetrics, envFlag, planAdjudication, adjudicateRun, assertJudgeAllowed, cellsFromResults, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -131,11 +131,12 @@ export async function serveReview(opts) {
                     const rr = await regradeScenario({
                         runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
                         specDir: dirname(specPath), threshold, mode: results.mode,
+                        expectedReps: prev.reps ?? 1,
                     });
                     const merged = results.scenarios.map((s) => 
                     // Same contract as `grade`, through the same choke point.
                     s.id === body.scenarioId
-                        ? rebuildScenarioResult(rr, s, { objective: "carry", adjudication: "drop" })
+                        ? rebuildScenarioResult({ ...rr, metrics: mergeScenarioMetrics(s.metrics, rr.metrics) }, s, { objective: "carry", adjudication: "drop" })
                         : s);
                     const written = writeResults(column.runDir, {
                         skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.8.0",
+    "version": "0.9.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.8.0",
-        "@skill-harness/adapters": "0.8.0"
+        "@skill-harness/core": "0.9.0",
+        "@skill-harness/adapters": "0.9.0"
     }
 }