@skill-harness/core 0.29.0 → 0.31.0

package/dist/run.js

@@ -1,13 +1,15 @@
 import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { sourceHashes, sourceHashRoots } from "./sources.js";
-import { judgeResemblesSubject } from "./grade.js";
+import { judgeResemblesSubject, resolveJudgeReferences } from "./grade.js";
+import { inspectEvalAuthoring } from "./authoring-preflight.js";
 import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { evaluateOutputGates, hasOutputGates, runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { executionFacts, transcriptResponses } from "./judge-evidence.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
@@ -17,6 +19,8 @@ import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
 import { providerFailureFromTranscript, executionFailureFromTranscript, withExecutionFailure } from "./provider-failure.js";
+import { withResourceMonitoring } from "./resource-monitor.js";
+import { throwIfProcessAborted, isProcessControlError } from "./util/process.js";
 /**
  * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
  * `PI_GRANTS_LEDGER` at exactly this filename inside the run directory. A fixed
@@ -52,7 +56,10 @@ export async function runSkillModel(opts) {
     // --only: validate against the spec BEFORE spending anything — a typo'd id must not
     // silently run zero scenarios and report success.
     let scenarios = spec.scenarios;
-    const partial = Boolean(opts.only && opts.only.length > 0);
+    const partial = opts.only !== undefined;
+    if (partial && (!opts.only.length || opts.only.some(id => !id.trim() || id !== id.trim()) || new Set(opts.only).size !== opts.only.length)) {
+        throw new Error("--only requires a nonempty, unique scenario ID list");
+    }
     if (partial) {
         const known = new Set(spec.scenarios.map((s) => s.id));
         const unknown = opts.only.filter((id) => !known.has(id));
@@ -63,6 +70,11 @@ export async function runSkillModel(opts) {
         scenarios = spec.scenarios.filter((s) => wanted.has(s.id));
         log(`  --only ${opts.only.join(",")} — partial run, will not be ship-graded`);
     }
+    const authoringErrors = inspectEvalAuthoring({ ...spec, scenarios }).filter(f => f.severity === "error");
+    if (authoringErrors.length)
+        throw new Error(authoringErrors.map(f => `${f.scenario}: ${f.message}`).join("\n"));
+    const judgeReferences = Object.fromEntries(scenarios.map(s => [s.id, resolveJudgeReferences(s, dirname(opts.specPath))]));
+    const hashes = sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona, judgeReferences });
     if (judgeResemblesSubject(judge, model)) {
         log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
@@ -71,119 +83,124 @@ export async function runSkillModel(opts) {
     const runDir = runDirFor(dirname(opts.testsDir ?? join(skillDir, "tests")), adapter.name, model, timestamp, arm.name);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
-    // Which harness CLI delivered the skill, asked once per run and recorded with the
-    // numbers. A pi upgrade (0.80.x → 0.83.0) silently changed what green mode
-    // measures, and the incident was invisible in the artifacts because nothing wrote
-    // this down. Never fatal: `null` means the adapter couldn't say.
-    const harnessCliVersion = (await adapter.version?.()) ?? null;
-    appendJournal(runDir, {
-        event: "run-started", ts: now(),
-        skill: spec.skill, harness: adapter.name, model: opts.modelToken,
-        harness_cli_version: harnessCliVersion,
-        judge: { provider: judge.provider, model: judge.model },
-        mode, label: opts.label ?? null,
-    });
-    // The canary spends one probe before the wave, so a run that isn't measuring the
-    // skill dies for the price of a rep instead of producing a plausible scorecard.
-    // Green only: red delivers nothing by design, and force delivers through the
-    // system prompt, which needs no probe.
-    let canaryStatus = null;
-    if (opts.canary && mode !== "green") {
-        // Ignoring a flag silently is a small version of the bug this whole feature is
-        // about. Say it, and say why it isn't needed.
-        log(`  --canary ignored in mode=${mode} — ${mode === "force" ? "the system prompt delivers the skill unconditionally" : "a baseline delivers no skill by design"}`);
-    }
-    if (opts.canary && mode === "green") {
-        const probeCwd = createWorkspace("none", { specDir: dirname(opts.specPath) });
-        let canary;
-        try {
-            canary = await runDeliveryCanary({
-                adapter, model, skillDir, skillName: spec.skill, cwd: probeCwd.cwd,
-            });
-        }
-        finally {
-            probeCwd.cleanup();
-        }
+    return withResourceMonitoring(runDir, async () => {
+        throwIfProcessAborted();
+        // Which harness CLI delivered the skill, asked once per run and recorded with the
+        // numbers. A pi upgrade (0.80.x → 0.83.0) silently changed what green mode
+        // measures, and the incident was invisible in the artifacts because nothing wrote
+        // this down. Never fatal: `null` means the adapter couldn't say.
+        const harnessCliVersion = (await adapter.version?.()) ?? null;
         appendJournal(runDir, {
-            event: "delivery-canary", ts: now(),
-            status: canary.status, anchor: canary.anchor, detail: canary.detail,
+            event: "run-started", ts: now(),
+            skill: spec.skill, harness: adapter.name, model: opts.modelToken,
+            harness_cli_version: harnessCliVersion,
+            judge: { provider: judge.provider, model: judge.model },
+            mode, label: opts.label ?? null,
         });
-        if (canary.status === "fail")
-            throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
-        if (canary.status === "skipped") {
-            // Recorded, not just logged: `journal.jsonl` is gitignored, and the claim
-            // "this run's delivery was verified" has to survive a commit — including
-            // when it is the claim that it wasn't.
-            canaryStatus = "skipped";
-            log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        // One optional instructed loading probe can stop a wave whose delivery check
+        // is inconclusive. It does not measure autonomous selection in later scenarios.
+        // Green only: red delivers nothing by design, and force delivers through the
+        // system prompt, which needs no probe.
+        let canaryStatus = null;
+        let canaryObservations;
+        if (opts.canary && mode !== "green") {
+            // Ignoring a flag silently is a small version of the bug this whole feature is
+            // about. Say it, and say why it isn't needed.
+            log(`  --canary ignored in mode=${mode} — ${mode === "force" ? "the system prompt delivers the skill unconditionally" : "a baseline delivers no skill by design"}`);
         }
-        else {
-            canaryStatus = "pass";
-            log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
+        if (opts.canary && mode === "green") {
+            const probeCwd = createWorkspace("none", { specDir: dirname(opts.specPath) });
+            let canary;
+            try {
+                canary = await runDeliveryCanary({
+                    adapter, model, skillDir, skillName: spec.skill, cwd: probeCwd.cwd,
+                });
+            }
+            finally {
+                probeCwd.cleanup();
+            }
+            appendJournal(runDir, {
+                event: "delivery-canary", ts: now(),
+                status: canary.status, anchor: canary.anchor, detail: canary.detail,
+                observations: canary.observations, probeCalls: canary.probeCalls,
+            });
+            canaryObservations = { ...canary.observations, probeCalls: canary.probeCalls };
+            if (canary.status === "fail")
+                throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
+            if (canary.status === "skipped") {
+                // Recorded, not just logged: `journal.jsonl` is gitignored, and the claim
+                // requested delivery observation has to survive a commit, including a skip.
+                canaryStatus = "skipped";
+                log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+            }
+            else {
+                canaryStatus = "pass";
+                log(`  ✓ delivery canary — ${canary.detail}`);
+            }
         }
-    }
-    // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
-    const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
-    const owners = [];
-    const tasks = [];
-    const armDefinitions = { count: 0 };
-    scenarios.forEach((scenario, si) => {
-        for (let k = 0; k < repCounts[si]; k++) {
-            const rep = k;
-            const total = repCounts[si];
-            owners.push(si);
-            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions }));
+        // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
+        const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
+        const owners = [];
+        const tasks = [];
+        const armDefinitions = { count: 0 };
+        scenarios.forEach((scenario, si) => {
+            for (let k = 0; k < repCounts[si]; k++) {
+                const rep = k;
+                const total = repCounts[si];
+                owners.push(si);
+                tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions, judgeReferences }));
+            }
+        });
+        const flat = await runPool(tasks, opts.concurrency ?? 1);
+        const grouped = scenarios.map(() => []);
+        flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
+        const scenarioResults = scenarios.map((scenario, si) => {
+            const threshold = scenario.critical
+                ? effectiveThreshold(undefined, scenario)
+                : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
+            return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+        });
+        const ctx = scoreContextFor({ mode, partial }, spec);
+        const results = writeResults(runDir, {
+            skill: spec.skill,
+            harness: adapter.name,
+            harness_cli_version: harnessCliVersion ?? undefined,
+            delivery_canary: canaryStatus ?? undefined,
+            delivery_observations: canaryObservations,
+            model: opts.modelToken,
+            judge: { provider: judge.provider, model: judge.model },
+            timestamp,
+            label: opts.label ?? null,
+            mode,
+            ...(partial ? { partial: true } : {}),
+            // Only the scenarios this run actually measured: a --only run must not claim
+            // coverage of scenarios it skipped.
+            source_hashes: hashes,
+            ...(opts.recordSourceRoots ? { source_hash_roots: sourceHashRoots(hashes) } : {}),
+            scenarios: scenarioResults,
+            ...(arm.name === NONE_ARM.name ? {} : {
+                arm: {
+                    name: arm.name,
+                    extensions: arm.extensions,
+                    definitions: armDefinitions.count,
+                    ledger_events: countLedgerEvents(runDir),
+                    // The DECLARED env, `<run-dir>` left unsubstituted. It is the condition
+                    // being measured (grant, max depth), so leaving it out made two runs at
+                    // different settings byte-identical here and inside the same `+<arm>`
+                    // tag — `stability` then reads the verdict difference between two
+                    // conditions as one lineage flipping. The substituted form would be the
+                    // opposite error: it embeds this run's temp path, so re-running the SAME
+                    // condition would record two different-looking arms.
+                    env: arm.env,
+                },
+            }),
+        }, ctx);
+        if (ctx) {
+            const g = results.effective_grade;
+            appendJournal(runDir, { event: "score", ts: now(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
         }
+        return { runDir, results };
     });
-    const flat = await runPool(tasks, opts.concurrency ?? 1);
-    const grouped = scenarios.map(() => []);
-    flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
-    const scenarioResults = scenarios.map((scenario, si) => {
-        const threshold = scenario.critical
-            ? effectiveThreshold(undefined, scenario)
-            : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
-        return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
-    });
-    const ctx = scoreContextFor({ mode, partial }, spec);
-    const hashes = sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona });
-    const results = writeResults(runDir, {
-        skill: spec.skill,
-        harness: adapter.name,
-        harness_cli_version: harnessCliVersion ?? undefined,
-        delivery_canary: canaryStatus ?? undefined,
-        model: opts.modelToken,
-        judge: { provider: judge.provider, model: judge.model },
-        timestamp,
-        label: opts.label ?? null,
-        mode,
-        ...(partial ? { partial: true } : {}),
-        // Only the scenarios this run actually measured: a --only run must not claim
-        // coverage of scenarios it skipped.
-        source_hashes: hashes,
-        ...(opts.recordSourceRoots ? { source_hash_roots: sourceHashRoots(hashes) } : {}),
-        scenarios: scenarioResults,
-        ...(arm.name === NONE_ARM.name ? {} : {
-            arm: {
-                name: arm.name,
-                extensions: arm.extensions,
-                definitions: armDefinitions.count,
-                ledger_events: countLedgerEvents(runDir),
-                // The DECLARED env, `<run-dir>` left unsubstituted. It is the condition
-                // being measured (grant, max depth), so leaving it out made two runs at
-                // different settings byte-identical here and inside the same `+<arm>`
-                // tag — `stability` then reads the verdict difference between two
-                // conditions as one lineage flipping. The substituted form would be the
-                // opposite error: it embeds this run's temp path, so re-running the SAME
-                // condition would record two different-looking arms.
-                env: arm.env,
-            },
-        }),
-    }, ctx);
-    if (ctx) {
-        const g = results.effective_grade;
-        appendJournal(runDir, { event: "score", ts: now(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
-    }
-    return { runDir, results };
 }
 /**
  * True when any assistant turn in a transcript is blank — the shape a harness timeout
@@ -282,6 +299,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
             // An adapter that THREW is the same class of event and takes the same retry.
             for (let attempt = 0; attempt < 2; attempt++) {
+                throwIfProcessAborted();
                 if (attempt > 0) {
                     const why = adapterFailure ? `adapter failed (${adapterFailure})` : "empty response";
                     appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, reason: why, ...repField });
@@ -332,6 +350,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                     else {
                         const req = {
                             skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                            tools: scenario.tools,
                             // resolved like fixtures: relative to the spec's dir
                             systemPromptFile: scenario.systemPromptFile
                                 ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
@@ -361,6 +380,9 @@ async function runRep(scenario, rep, repCount, ctx) {
                     }
                 }
                 catch (e) {
+                    if (isProcessControlError(e))
+                        throw e;
+                    throwIfProcessAborted();
                     // An adapter that THREW is one rep's infrastructure failure, not the
                     // wave's. `runPool` is fail-fast and `runRep` had only a `finally`, so
                     // a single pi crash or `PI_TIMEOUT_MS` timeout unwound the whole run:
@@ -461,23 +483,26 @@ async function runRep(scenario, rep, repCount, ctx) {
         // timeout that caused it, and it is that text, not the real reason, that would
         // reach the results record. The verdict is ERROR either way.
         let objective;
+        if (traces.length > 0)
+            writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+        const execution = executionFacts(traces, {
+            scenarioId: scenario.id, mode, repetition: rep, turns: scenario.turns.length,
+            subject: ctx.model, responses: transcriptResponses(transcript, scenario),
+        }, { source: "live" });
         if ((scenario.traceAssert || hasOutputGates(scenario)) && !adapterFailure && !noResponse && !infrastructureFailure) {
             const assertionResults = [...outputGate.assertions];
             let status = outputGate.status;
             let traceMeta = {};
             const outputMeta = outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {};
             if (scenario.traceAssert) {
-                if (traces.length > 0) {
-                    writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
-                }
-                const merged = mergeTraces(traces);
+                const merged = execution.status === "invalid" ? null : mergeTraces(traces);
                 if (unobservablePaths) {
                     status = "ERROR";
                     assertionResults.push({ kind: "unchanged_path", status: "ERROR", detail: "workspace changes could not be observed" });
                 }
                 else if (merged === null) {
                     status = "ERROR";
-                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: "no execution trace was produced" });
+                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: execution.status === "invalid" ? execution.reason : "no execution trace was produced" });
                 }
                 else {
                     const gate = evaluateTraceGates(scenario.traceAssert, merged);
@@ -534,6 +559,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             const o = await judgeOneRep({
                 runDir, spec, scenario, transcript, adapter: ctx.adapter, judge,
                 specDir: dirname(ctx.specPath), mode, rep: repCount > 1 ? rep : undefined, now,
+                execution, objective, references: ctx.judgeReferences[scenario.id],
             });
             verdict = o.verdict;
             reason = o.reason;
@@ -542,7 +568,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             judgeCalls = o.metrics?.judge_calls ?? 0;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        const subject = mergeTraces(traces)?.metrics;
+        const subject = execution.status === "invalid" ? undefined : mergeTraces(traces)?.metrics;
         return {
             verdict, reason, suspect, objective, judgment,
             metrics: {
@@ -632,7 +658,12 @@ export function formatScorecard(summary, lift, stability) {
         lines.push(`  NOTE:  green delivery is harness-version-dependent` +
             (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
             ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
-            ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
+            ` Use --mode force for explicit instruction delivery, or --canary for one instructed loading check; autonomous selection remains untested.`);
+    }
+    if (results.mode === "green" && results.delivery_canary === "pass") {
+        const observation = results.delivery_observations;
+        lines.push(`  DELIVERY: instructed probe read ${observation?.successfulSkillRead ?? "unobserved"}; ` +
+            `autonomous skill selection in scenarios was not measured by the probe.`);
     }
     // Boundary cells last, because they qualify the verdicts above: a ✓ on a cell that
     // flipped between the last two runs is one draw, whatever its rep count said.

package/dist/saved-run.d.ts

@@ -0,0 +1,15 @@
+import type { Spec } from "./spec.js";
+export interface SavedAnswerPlan {
+    providerCalls: 0;
+    resultsChanged: false;
+    scenarios: {
+        id: string;
+        action: "reuse" | "regrade" | "regate" | "rescore" | "rerun" | "inspect";
+        changed: string[];
+        note: string;
+    }[];
+}
+/** This is an input-identity plan, never an approval or a claim that a failed answer improved. */
+export declare function planSavedAnswers(runDir: string, spec: Spec, skillDir: string, specDir: string): SavedAnswerPlan;
+/** Copy a selected run, with limits and byte identities, before revising its judgments. */
+export declare function forkSavedRun(source: string, destination: string): string;

package/dist/saved-run.js

@@ -0,0 +1,114 @@
+import { createHash } from "node:crypto";
+import { lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, writeFileSync } from "node:fs";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
+import { readResults, findTranscriptFiles, transcriptPath } from "./results.js";
+import { currentHashFor, isSupersededKey, sourceHashes } from "./sources.js";
+/** This is an input-identity plan, never an approval or a claim that a failed answer improved. */
+export function planSavedAnswers(runDir, spec, skillDir, specDir) {
+    const previous = readResults(runDir);
+    const context = { skillDir, specDir, scenarios: spec.scenarios, judgePersona: spec.judge_persona };
+    const recorded = previous.source_hashes;
+    const current = sourceHashes(context);
+    const changed = Object.keys({ ...recorded, ...current }).filter(key => {
+        if (isSupersededKey(key, { ...recorded, ...current }))
+            return false;
+        return !recorded || !(key in recorded) || currentHashFor(key, context) !== recorded[key];
+    });
+    return { providerCalls: 0, resultsChanged: false, scenarios: spec.scenarios.map(s => {
+            const prior = previous.scenarios.find(p => p.id === s.id);
+            const relevant = changed.filter(key => {
+                if (/^(stimulus|rubric|policy|gates|scenario):/.test(key) && key !== "rubric:__persona")
+                    return key.slice(key.indexOf(":") + 1) === s.id;
+                if (key.startsWith("fixture:"))
+                    return key.slice(8) === (typeof s.workspace === "object" ? s.workspace.fixture : s.fixture);
+                if (key.startsWith("prompt:"))
+                    return key.slice(7) === s.systemPromptFile;
+                if (!key.includes(":") && key !== "SKILL.md")
+                    return [s.systemPromptFile, s.assert?.post_test, ...(s.extensions ?? [])].includes(key);
+                return true;
+            });
+            const base = { id: s.id, changed: relevant };
+            if (!prior)
+                return { ...base, action: "rerun", note: "No saved answer exists for this scenario." };
+            const files = findTranscriptFiles(runDir, s.id, previous.mode);
+            const reps = prior.reps ?? 1;
+            const complete = Number.isSafeInteger(reps) && reps > 0 && files.length === reps && files.every((file, rep) => {
+                if (file !== basename(transcriptPath(runDir, s.id, previous.mode, reps === 1 ? undefined : rep)))
+                    return false;
+                try {
+                    const stat = lstatSync(join(runDir, file));
+                    return stat.isFile() && !stat.isSymbolicLink() && stat.size > 0;
+                }
+                catch {
+                    return false;
+                }
+            });
+            if (!complete)
+                return { ...base, action: "inspect", note: "Saved answers are incomplete; recover original evidence before planning paid work." };
+            if (!recorded || !recorded[`stimulus:${s.id}`])
+                return { ...base, action: "inspect", note: "Input identity is incomplete; do not assume a saved answer matches this task." };
+            if (relevant.some(key => key in recorded && currentHashFor(key, context) === undefined))
+                return { ...base, action: "inspect", note: "An input identity cannot be interpreted by this harness; inspect the original evidence before planning a rerun." };
+            if (prior.judge_verdict === "ERROR")
+                return { ...base, action: "inspect", note: "Resolve transport/capture errors first; recover saved answers before considering another subject run." };
+            const stimulusChanged = relevant.some(key => !/^(rubric|gates|policy):/.test(key));
+            if (stimulusChanged)
+                return { ...base, action: "rerun", note: "Subject inputs or skill changed. Old answers remain historical and cannot establish current behavior." };
+            if (relevant.some(k => k.startsWith("gates:")))
+                return { ...base, action: "regate", note: "Replay supported assertions on saved evidence; missing execution evidence remains unknown. Regrade too if the rubric changed." };
+            if (relevant.some(k => k.startsWith("rubric:")))
+                return { ...base, action: "regrade", note: "Judge the saved answers against corrected references/rubric; no subject call is needed." };
+            if (relevant.some(k => k.startsWith("policy:")))
+                return { ...base, action: "rescore", note: "Recompute policy using retained verdicts; no provider calls." };
+            return { ...base, action: "reuse", note: "Inputs match. Retain the existing verdict, including failures; this is not a new stability sample." };
+        }) };
+}
+/** Copy a selected run, with limits and byte identities, before revising its judgments. */
+export function forkSavedRun(source, destination) {
+    const root = realpathSync(source), out = join(realpathSync(dirname(resolve(destination))), basename(destination));
+    const rel = relative(root, out);
+    if (!rel || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith("../") && !rel.startsWith("..\\")))
+        throw new Error("Saved-run output must be outside the original run");
+    const files = [];
+    let bytes = 0, entries = 0;
+    const sha = (data) => createHash("sha256").update(data).digest("hex");
+    const visit = (dir, prefix, depth) => {
+        if (depth > 16)
+            throw new Error("Saved-run copy exceeds directory depth limit");
+        for (const entry of readdirSync(dir, { withFileTypes: true })) {
+            if (++entries > 4096)
+                throw new Error("Saved-run copy exceeds 4096 entries");
+            const path = join(dir, entry.name), key = prefix + entry.name, stat = lstatSync(path);
+            if (stat.isSymbolicLink())
+                throw new Error("Saved-run copy refuses symbolic links");
+            if (stat.isDirectory())
+                visit(path, key + "/", depth + 1);
+            else if (stat.isFile()) {
+                bytes += stat.size;
+                if (stat.size > 64 * 1024 * 1024 || bytes > 256 * 1024 * 1024)
+                    throw new Error("Saved-run copy exceeds evidence byte limit; select a smaller run");
+                files.push({ path: key, bytes: stat.size, sha256: sha(readFileSync(path)) });
+            }
+            else
+                throw new Error("Saved-run copy requires regular files");
+        }
+    };
+    visit(root, "", 0);
+    if (!files.some(f => f.path === "results.yaml"))
+        throw new Error("Saved run lacks results.yaml");
+    mkdirSync(out, { mode: 0o700 }); // Exclusive: never overwrite another result directory.
+    for (const file of files) {
+        const data = readFileSync(join(root, file.path));
+        if (sha(data) !== file.sha256)
+            throw new Error("Saved evidence changed during copy; output is incomplete");
+        const target = join(out, file.path);
+        mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
+        writeFileSync(target, data, { flag: "wx", mode: 0o600 });
+    }
+    for (const file of files)
+        if (sha(readFileSync(join(root, file.path))) !== file.sha256 || sha(readFileSync(join(out, file.path))) !== file.sha256)
+            throw new Error("Saved evidence verification failed; output is incomplete");
+    writeFileSync(join(out, `reuse-source-${Date.now()}.json`), JSON.stringify({ schema: 1, purpose: "saved-answer-regrade", source: root, files, subjectCalls: 0 }, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+    return out;
+}
+//# sourceMappingURL=saved-run.js.map

package/dist/scheduler.d.ts

@@ -2,10 +2,8 @@
  * Run `tasks` with at most `concurrency` thunks in flight at once, returning
  * their results in input order (not completion order). `concurrency <= 1` runs
  * them strictly sequentially — identical to a plain for-await loop. A thunk that
- * throws rejects the returned promise (fail-fast) — once any task rejects,
- * `runPool` rejects immediately. Tasks already claimed by other workers still
- * run to completion (JS has no cancellation), and sibling workers may pull
- * further tasks before the rejection unwinds; the guarantee is that runPool
- * rejects, not that dispatch halts.
+ * throws stops new dispatch; already claimed tasks settle before rejection.
+ * A surrounding process scope supplies cancellation to active subprocesses.
+ * Waiting for settlement prevents reporting a stopped run while its workers remain active.
  */
 export declare function runPool<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]>;

package/dist/scheduler.js

@@ -2,26 +2,38 @@
  * Run `tasks` with at most `concurrency` thunks in flight at once, returning
  * their results in input order (not completion order). `concurrency <= 1` runs
  * them strictly sequentially — identical to a plain for-await loop. A thunk that
- * throws rejects the returned promise (fail-fast) — once any task rejects,
- * `runPool` rejects immediately. Tasks already claimed by other workers still
- * run to completion (JS has no cancellation), and sibling workers may pull
- * further tasks before the rejection unwinds; the guarantee is that runPool
- * rejects, not that dispatch halts.
+ * throws stops new dispatch; already claimed tasks settle before rejection.
+ * A surrounding process scope supplies cancellation to active subprocesses.
+ * Waiting for settlement prevents reporting a stopped run while its workers remain active.
  */
 export async function runPool(tasks, concurrency) {
     const limit = Math.max(1, Math.floor(concurrency));
     const results = new Array(tasks.length);
     let next = 0;
+    let failure;
+    let failed = false;
     async function worker() {
         while (true) {
+            if (failed)
+                return;
             const i = next++;
             if (i >= tasks.length)
                 return;
-            results[i] = await tasks[i]();
+            try {
+                results[i] = await tasks[i]();
+            }
+            catch (error) {
+                if (!failed)
+                    failure = error;
+                failed = true;
+                return;
+            }
         }
     }
     const workerCount = Math.min(limit, tasks.length);
     await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    if (failed)
+        throw failure;
     return results;
 }
 //# sourceMappingURL=scheduler.js.map

package/dist/seeded.js

@@ -234,6 +234,7 @@ export async function runSeeded(scenario, opts) {
         model: opts.model,
         mode: opts.mode,
         turns: scenario.turns,
+        tools: scenario.tools,
         cwd: repo,
         // Resolved against the spec dir, exactly like fixtures and post-tests —
         // the arm's extensions (already absolute) join alongside, same as the

package/dist/sources.d.ts

@@ -1,4 +1,5 @@
 import type { Scenario } from "./spec.js";
+import { type JudgeReference } from "./judge-references.js";
 export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
 /**
  * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
@@ -103,7 +104,7 @@ export declare function fileSha256(path: string): string | null;
  */
 export declare function dirSha256(dir: string): string | null;
 export declare function stimulusDigest(s: Scenario): string;
-export declare function rubricDigest(s: Scenario): string;
+export declare function rubricDigest(s: Scenario, specDir?: string, references?: JudgeReference[]): string;
 export declare function policyDigest(s: Scenario): string;
 /** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export declare function gatesDigest(s: Scenario): string | null;
@@ -143,6 +144,8 @@ export interface SourceContext {
      * caller instead of leaving one quietly un-checked.
      */
     judgePersona: string;
+    /** Snapshot resolved before execution, shared by prompt construction and recorded rubric hashes. */
+    judgeReferences?: Record<string, JudgeReference[]>;
 }
 /**
  * Hash every source this run measures: SKILL.md, each distinct