@skill-harness/core 0.28.0 → 0.30.0

package/dist/judge-evidence.js

@@ -0,0 +1,120 @@
+import { existsSync, readFileSync } from "node:fs";
+import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
+import { deserializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { redactText } from "./redaction.js";
+import { finalAssistantMessage } from "./seeded.js";
+const HASH = /^[a-f0-9]{64}$/;
+const MAX_CALLS = 32;
+const object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+const index = (value) => Number.isSafeInteger(value) && value >= 0;
+/** Validate the fields consumed by merging and the evidence projection before using them. */
+function validTrace(trace) {
+    if (!object(trace) || trace.trace_version !== EXECUTION_TRACE_VERSION || !object(trace.subject) || typeof trace.subject.provider !== "string" || typeof trace.subject.model !== "string" ||
+        !(trace.pi_version === null || typeof trace.pi_version === "string") || typeof trace.final_text !== "string" ||
+        !index(trace.turn) || !index(trace.rep) || !Array.isArray(trace.tool_calls) ||
+        !(trace.changed_paths === null || Array.isArray(trace.changed_paths) && trace.changed_paths.every(p => typeof p === "string")) ||
+        !(trace.cost_usd === null || typeof trace.cost_usd === "number" && Number.isFinite(trace.cost_usd)) ||
+        !(trace.capture_errors === undefined || Array.isArray(trace.capture_errors) && trace.capture_errors.every(e => typeof e === "string")))
+        return false;
+    if (trace.metrics !== undefined && (!object(trace.metrics) ||
+        ["input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "cost_usd"].some(key => {
+            const value = trace.metrics[key];
+            return value !== null && !(typeof value === "number" && Number.isFinite(value) && value >= 0);
+        }) || [trace.metrics.tool_calls, trace.metrics.delegated_children, trace.metrics.max_concurrency].some(value => !index(value))))
+        return false;
+    const ids = new Set();
+    const completions = new Set();
+    for (const [position, call] of trace.tool_calls.entries()) {
+        if (!object(call) || typeof call.id !== "string" || !call.id || ids.has(call.id) ||
+            typeof call.name !== "string" || !call.name || !object(call.args) || call.issueIndex !== position ||
+            !(call.completionIndex === -1 || index(call.completionIndex)) || typeof call.isError !== "boolean" ||
+            !object(call.result) || !index(call.result.bytes) || typeof call.result.sha256 !== "string" || !HASH.test(call.result.sha256))
+            return false;
+        ids.add(call.id);
+        if (call.completionIndex >= 0) {
+            if (completions.has(call.completionIndex))
+                return false;
+            completions.add(call.completionIndex);
+        }
+    }
+    return [...completions].sort((a, b) => a - b).every((value, position) => value === position);
+}
+export function unavailableExecution(reason = "No bound execution trace was supplied.", status = "unknown") {
+    return { schema: 1, status, coverage: "unknown", reason, observedCalls: null, omittedCalls: 0, calls: [] };
+}
+/** Tool inputs are untrusted labels. Keep only bounded local read paths, never commands, URLs, tasks or results. */
+function readPath(call) {
+    if (call.name !== "read")
+        return undefined;
+    const path = call.args.path ?? call.args.file_path;
+    if (typeof path !== "string")
+        return undefined;
+    if (path.length > 256 || /[\x00-\x1f\x7f?#]/.test(path) || /^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+        return "[path withheld]";
+    return redactText(path);
+}
+/** Digest binding establishes artifact continuity, not a signature or proof of file contents. */
+export function executionFacts(traces, identity, binding) {
+    if (binding.source === "saved" && !binding.expectedSha256)
+        return unavailableExecution("Legacy trace has no recorded digest binding; execution is unverified.");
+    const invalid = (reason) => unavailableExecution(reason, "invalid");
+    if (!traces.length)
+        return binding.source === "live" ? unavailableExecution() : invalid("Recorded execution trace is missing or unreadable.");
+    if (traces.some(trace => !validTrace(trace)))
+        return invalid("Execution trace has malformed structure.");
+    if (traces.some((trace, turn) => trace.scenario_id !== identity.scenarioId || trace.mode !== identity.mode || trace.rep !== identity.repetition || trace.turn !== turn ||
+        identity.subject && (trace.subject.provider !== identity.subject.provider || trace.subject.model !== identity.subject.model) ||
+        identity.piVersion !== undefined && trace.pi_version !== identity.piVersion ||
+        trace.subject.provider !== traces[0].subject.provider || trace.subject.model !== traces[0].subject.model || trace.pi_version !== traces[0].pi_version)) {
+        return invalid("Execution trace identity or turn sequence does not match this repetition.");
+    }
+    if (identity.responses !== undefined && (identity.responses === null || identity.responses.length !== traces.length || traces.some((trace, turn) => trace.final_text.trim() !== identity.responses[turn]))) {
+        return invalid("Execution trace final answers do not match the judged transcript.");
+    }
+    if (traces.some(trace => trace.trace_sha256 !== undefined && trace.trace_sha256 !== traceSha256(trace)))
+        return invalid("Execution trace has a mismatching turn digest.");
+    // Use the existing canonical merge/digest; only the projection below reaches the judge.
+    const merged = mergeTraces(traces);
+    const digest = merged?.trace_sha256;
+    if (!merged || !digest)
+        return invalid("Execution trace could not be merged.");
+    if (binding.source === "saved" && binding.expectedSha256 !== digest)
+        return invalid("Saved trace no longer matches the digest recorded by the run.");
+    const complete = traces.length === identity.turns && traces.every(trace => trace.final_status === "complete" && !trace.capture_errors?.length && trace.tool_calls.every(call => call.completionIndex >= 0));
+    const calls = merged.tool_calls.slice(0, MAX_CALLS).map(call => ({
+        tool: /^[A-Za-z0-9_.:/-]{1,80}$/.test(call.name) ? call.name : "[tool name withheld]",
+        state: call.completionIndex < 0 ? "unfinished" : call.isError ? "error" : "completed",
+        ...(readPath(call) === undefined ? {} : { path: readPath(call) }),
+        ...(call.completionIndex < 0 ? {} : { resultBytes: call.result.bytes, resultSha256: call.result.sha256 }),
+    }));
+    const omittedCalls = merged.tool_calls.length - calls.length;
+    return {
+        schema: 1, status: "verified", coverage: complete && !omittedCalls ? "complete" : "partial",
+        reason: complete ? omittedCalls ? "Call list is bounded; omitted calls may include relevant actions." : "All expected turns settled; all recorded calls are listed." : "Turn settlement or capture coverage is incomplete; do not infer that unlisted actions did not occur.",
+        traceSha256: digest, observedCalls: merged.tool_calls.length, omittedCalls, calls,
+    };
+}
+/** Reuse the transcript ambiguity check, then compare every assistant turn, excluding harness-owned trailers. */
+export function transcriptResponses(transcript, scenario) {
+    if (finalAssistantMessage(transcript, scenario) === null)
+        return null;
+    return transcript.split(/^<<< ASSISTANT:[ \t]*$/m).slice(1).map(section => section.split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*)$/m)[0].trim());
+}
+/** Read every JSONL turn or decline the whole artifact; never silently drop malformed lines. */
+export function loadExecutionFacts(path, identity, expectedSha256) {
+    if (!expectedSha256)
+        return unavailableExecution("Legacy run has no recorded trace digest; execution is unverified.");
+    if (!existsSync(path))
+        return unavailableExecution("Recorded execution trace is missing.", "invalid");
+    try {
+        const lines = readFileSync(path, "utf8").split("\n").filter(line => line.trim());
+        const traces = lines.map(deserializeTrace);
+        if (traces.some(trace => trace === null))
+            return unavailableExecution("Execution trace contains unreadable turns.", "invalid");
+        return executionFacts(traces, identity, { source: "saved", expectedSha256 });
+    }
+    catch {
+        return unavailableExecution("Execution trace could not be read or validated.", "invalid");
+    }
+}
+//# sourceMappingURL=judge-evidence.js.map

package/dist/judge-references.d.ts

@@ -0,0 +1,8 @@
+import type { Scenario } from "./spec.js";
+export interface JudgeReference {
+    file: string;
+    sha256: string;
+    text: string;
+}
+/** Only explicitly selected local UTF-8 files are read. Never discover sibling files. */
+export declare function resolveJudgeReferences(scenario: Scenario, specDir: string): JudgeReference[];

package/dist/judge-references.js

@@ -0,0 +1,34 @@
+import { createHash } from "node:crypto";
+import { readFileSync, realpathSync, statSync } from "node:fs";
+import { isAbsolute, relative, resolve } from "node:path";
+const LIMIT = 65_536;
+/** Only explicitly selected local UTF-8 files are read. Never discover sibling files. */
+export function resolveJudgeReferences(scenario, specDir) {
+    const selected = scenario.judgeReferences ?? [];
+    if (!selected.length)
+        return [];
+    if (selected.length > 16 || new Set(selected).size !== selected.length)
+        throw new Error("Invalid judge reference selection");
+    const root = realpathSync(specDir);
+    let total = 0;
+    return selected.map(file => {
+        if (!file || /[\\:\x00-\x1f]/.test(file) || file.startsWith("/") || file.split("/").some(part => !part || part === "." || part === ".."))
+            throw new Error(`Unsafe judge reference: ${file}`);
+        const path = realpathSync(resolve(root, file));
+        const rel = relative(root, path);
+        if (isAbsolute(rel) || rel === ".." || rel.startsWith("../") || rel.startsWith("..\\"))
+            throw new Error(`Judge reference escapes specification directory: ${file}`);
+        const info = statSync(path);
+        if (!info.isFile() || info.size > LIMIT)
+            throw new Error(`Judge reference must be a regular file of at most ${LIMIT} bytes: ${file}`);
+        const bytes = readFileSync(path);
+        total += bytes.length;
+        if (bytes.length > LIMIT || total > LIMIT)
+            throw new Error(`Selected judge references exceed ${LIMIT} bytes`);
+        const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        if (text.includes("\0"))
+            throw new Error(`Judge reference contains binary content: ${file}`);
+        return { file, sha256: createHash("sha256").update(bytes).digest("hex"), text };
+    });
+}
+//# sourceMappingURL=judge-references.js.map

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "covers" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "covers" | "lint-error" | "authoring";
 /**
  * How much a finding means.
  *

package/dist/lint.js

@@ -9,6 +9,8 @@ import { downgradeWarning } from "./downgrade.js";
 import { collectScoredRuns } from "./trends.js";
 import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
+import { fixtureInventory, inspectEvalAuthoring } from "./authoring-preflight.js";
+import { resolveJudgeReferences } from "./judge-references.js";
 /** True when a finding fails the gate. The single place the exit-code rule lives. */
 export function failsGate(f) {
     return (f.severity ?? "error") === "error";
@@ -49,6 +51,20 @@ export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
         return [{ skill: basename(skillDir), code: "spec", message }];
     }
     const skill = spec.skill;
+    for (const hint of inspectEvalAuthoring(spec, { fixtureFiles: scenario => {
+            const fixture = effectiveFixture(scenario);
+            if (!fixture)
+                return [];
+            // Unreadable/missing fixtures are reported by the fixture checks below.
+            try {
+                return fixtureInventory(resolve(testsDir, fixture));
+            }
+            catch {
+                return [];
+            }
+        } })) {
+        findings.push({ skill, scenario: hint.scenario, code: "authoring", severity: hint.severity ?? "info", message: `${hint.code}: ${hint.message}` });
+    }
     // A lint from an older tool than the records it is checking cannot be trusted to
     // check them: a 0.1.0 binary reported 38 spurious `consistency` findings against
     // partial runs that 0.3.x handles correctly, and it cannot read source-hash key
@@ -91,6 +107,12 @@ export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
     // relative to the spec's dir, matching workspace.ts resolve(specDir, fixture) where specDir = <skillDir>/tests.
     const specDir = dirname(specPath);
     for (const s of spec.scenarios) {
+        try {
+            resolveJudgeReferences(s, specDir);
+        }
+        catch (error) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: String(error) });
+        }
         const fx = effectiveFixture(s); // shared with sources.ts so hashing and linting can't drift
         if (fx) {
             const abs = isAbsolute(fx) ? fx : resolve(specDir, fx);

package/dist/regate.js

@@ -1,10 +1,12 @@
 import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
-import { parseVerdict, detectMisfire } from "./grade.js";
+import { parseModelRef } from "./adapters/types.js";
+import { parseVerdict, detectMisfire, resolveJudgeReferences } from "./grade.js";
 import { evaluateNeedleGates, evaluateOutputGates, hasNeedleGates, hasOutputGates } from "./seeded.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
 import { judgeOneRep } from "./regrade.js";
+import { loadExecutionFacts, transcriptResponses, unavailableExecution } from "./judge-evidence.js";
 import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, findDiffFiles, findTranscriptFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
@@ -150,6 +152,7 @@ export async function regateRun(opts) {
         throw new Error(`nothing to regate in ${opts.runDir}` +
             (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff/output assertions or assert.trace"));
     }
+    const references = Object.fromEntries(targets.map(s => [s.id, resolveJudgeReferences(s, opts.specDir)]));
     const changes = [];
     let judgeCalls = 0;
     const scenarios = [];
@@ -176,6 +179,16 @@ export async function regateRun(opts) {
             const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
             const outputGate = evaluateOutputGates(scenario, before);
+            const retainedPanel = rec.rep_judgments?.find(panel => panel.repetition === (rep ?? 0));
+            const recordedTraceHashes = [retainedPanel?.judgments[0]?.input?.trace_sha256, retainedPanel?.objective?.trace_sha256,
+                rec.objective?.rep_trace_sha256?.[rep ?? 0], rec.reps === undefined ? rec.objective?.trace_sha256 : undefined].filter((hash) => hash !== undefined);
+            const execution = new Set(recordedTraceHashes).size > 1
+                ? unavailableExecution("Conflicting recorded trace digests.", "invalid")
+                : loadExecutionFacts(tracePath(opts.runDir, scenario.id, mode, rep), {
+                    scenarioId: scenario.id, mode, repetition: rep ?? 0, turns: scenario.turns.length,
+                    ...(prev.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev.harness_cli_version,
+                    responses: transcriptResponses(before, scenario),
+                }, recordedTraceHashes[0]);
             // Output and trace gates are re-decided from saved artifacts. Free: no model, no judge.
             let traceFailure = null;
             const expectedOutputHash = rec.objective?.rep_output_sha256?.[rep ?? 0]
@@ -191,7 +204,11 @@ export async function regateRun(opts) {
                     assertions: [...outputIntegrityAssertion, ...outputGate.assertions],
                 }
                 : undefined;
-            if (scenario.traceAssert) {
+            if (execution.status === "invalid") {
+                traceFailure = `objective: ${execution.reason}`;
+                objective = { ...(objective ?? {}), status: "ERROR", assertions: [...(objective?.assertions ?? []), { kind: "trace_evidence", status: "ERROR", detail: execution.reason }] };
+            }
+            else if (scenario.traceAssert) {
                 const tp = tracePath(opts.runDir, scenario.id, mode, rep);
                 // A PARTIAL read is refused, not graded. `deserializeTrace` returns null
                 // for a malformed line and for a version it declines, and dropping those
@@ -303,7 +320,7 @@ export async function regateRun(opts) {
             const judged = await judgeOneRep({
                 runDir: opts.runDir, spec: opts.spec, scenario, transcript,
                 adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                mode, rep, now, rejudge: true,
+                mode, rep, now, rejudge: true, execution, objective, references: references[scenario.id],
             });
             outcomes.push({ ...judged, objective });
             judgeCalls += judged.metrics?.judge_calls ?? 0;

package/dist/regrade.d.ts

@@ -1,7 +1,9 @@
 import type { Spec, Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+import { type JudgeReference } from "./grade.js";
 import { type ScenarioResult, type ResultsFile } from "./results.js";
 import { type RepOutcome } from "./reps.js";
+import { type JudgeExecutionFacts } from "./judge-evidence.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -14,8 +16,9 @@ import { type RepOutcome } from "./reps.js";
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[]): Record<string, string> | undefined;
+export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[], specDir?: string, references?: Record<string, JudgeReference[]>): Record<string, string> | undefined;
 export interface RegradeOptions {
+    references?: JudgeReference[];
     runDir: string;
     spec: Spec;
     scenario: Scenario;
@@ -37,6 +40,8 @@ export interface RegradeOptions {
     /** Select only repetitions with retained ERROR criterion votes. */
     onlyUnparsed?: boolean;
     prior?: ScenarioResult;
+    subject?: ModelRef;
+    piVersion?: string;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -55,6 +60,8 @@ export declare function judgeOneRep(opts: {
     rejudge?: boolean;
     /** Retained objective evidence must precede missing-vote normalization. */
     objective?: RepOutcome["objective"];
+    execution?: JudgeExecutionFacts;
+    references?: JudgeReference[];
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no

package/dist/regrade.js

@@ -1,11 +1,14 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { buildJudgePrompt, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
+import { createHash } from "node:crypto";
+import { parseModelRef } from "./adapters/types.js";
+import { prepareJudgePrompt, resolveJudgeReferences, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, tracePath, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
 import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
+import { loadExecutionFacts, transcriptResponses } from "./judge-evidence.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -18,7 +21,7 @@ import { providerFailureFromTranscript, executionFailureFromTranscript } from ".
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export function refreshRubricHashes(recorded, spec, judgedIds) {
+export function refreshRubricHashes(recorded, spec, judgedIds, specDir, references) {
     if (!recorded)
         return undefined;
     const next = { ...recorded };
@@ -28,12 +31,27 @@ export function refreshRubricHashes(recorded, spec, judgedIds) {
         // Only refresh a key the run already carried: adding one for a scenario whose
         // rubric was never hashed would fabricate coverage.
         if (s && RUBRIC_PREFIX + id in next)
-            next[RUBRIC_PREFIX + id] = rubricDigest(s);
+            next[RUBRIC_PREFIX + id] = rubricDigest(s, specDir, references?.[id]);
     }
     if (PERSONA_KEY in next)
         next[PERSONA_KEY] = personaDigest(spec.judge_persona);
     return next;
 }
+function savedExecution(opts, mode, rep, transcript) {
+    const repetition = rep ?? 0;
+    const panel = opts.prior?.rep_judgments?.find(panel => panel.repetition === repetition);
+    const hashes = [panel?.judgments[0]?.input?.trace_sha256, panel?.objective?.trace_sha256, opts.prior?.objective?.rep_trace_sha256?.[repetition],
+        (opts.prior?.reps ?? 1) === 1 ? opts.prior?.objective?.trace_sha256 : undefined].filter((value) => value !== undefined);
+    if (new Set(hashes).size > 1)
+        throw new Error(`${opts.scenario.id}#${repetition}: conflicting recorded trace digests; regrade refused.`);
+    const execution = loadExecutionFacts(tracePath(opts.runDir, opts.scenario.id, mode, rep), {
+        scenarioId: opts.scenario.id, mode, repetition, turns: opts.scenario.turns.length,
+        subject: opts.subject, piVersion: opts.piVersion, responses: transcriptResponses(transcript, opts.scenario),
+    }, hashes[0]);
+    if (execution.status === "invalid")
+        throw new Error(`${opts.scenario.id}#${repetition}: ${execution.reason} Regrade refused; retained results were not replaced.`);
+    return execution;
+}
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
@@ -56,12 +74,30 @@ export async function judgeOneRep(opts) {
             metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 },
         };
     }
-    const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
+    if (opts.execution?.status === "invalid") {
+        const reason = `execution evidence: ${opts.execution.reason}`;
+        appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
+        return { verdict: "ERROR", reason, suspect: false, objective: opts.objective,
+            metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 } };
+    }
+    const prompt = prepareJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript, execution: opts.execution, references: opts.references }, specDir);
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
+    const inputBytes = JSON.stringify({ schema: 1, judge, execution: opts.execution ?? null, prompts: g.prompts }) + "\n";
+    const inputHash = createHash("sha256").update(inputBytes).digest("hex");
+    const artifact = `judge-input.${inputHash}.txt`;
+    const inputPath = join(runDir, artifact);
+    if (existsSync(inputPath)) {
+        if (readFileSync(inputPath, "utf8") !== inputBytes)
+            throw new Error("judge input artifact has conflicting bytes");
+    }
+    else
+        writeFileSync(inputPath, inputBytes, { encoding: "utf8", flag: "wx", mode: 0o600 });
+    const input = { artifact, sha256: inputHash, execution: opts.execution?.status === "verified" ? "verified" : "unknown",
+        ...(opts.execution?.traceSha256 ? { trace_sha256: opts.execution.traceSha256 } : {}) };
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), formatJudgeRawReplies(g.rawReplies), "utf8");
     const outcome = normalizeRepOutcome({
         verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, input, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),
@@ -81,6 +117,7 @@ export async function judgeOneRep(opts) {
  * persists).
  */
 export async function regradeScenario(opts) {
+    const references = opts.references ?? resolveJudgeReferences(opts.scenario, opts.specDir);
     const now = opts.now ?? (() => new Date().toISOString());
     const mode = opts.mode ?? "green";
     let files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
@@ -109,10 +146,12 @@ export async function regradeScenario(opts) {
         const file = files.find(file => (repIndexOf(file) ?? 0) === repetition);
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
+        const execution = savedExecution(opts, mode, rep, transcript);
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
             objective: panel?.objective ?? (expected === 1 ? opts.prior?.objective : undefined),
+            execution, references,
         }));
     }
     const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
@@ -214,6 +253,19 @@ export async function regradeRun(opts) {
         if (drifted.length)
             throw new Error(`criterion count changed for ${drifted.join(", ")} — use full grade instead of --unparsed-only`);
     }
+    // Refuse the whole selected batch before spending on any earlier cell. Recheck
+    // again immediately before each call so a later filesystem edit is not trusted.
+    for (const id of targets) {
+        const prior = recordedById.get(id);
+        for (const file of findTranscriptFiles(runDir, id, mode)) {
+            const rep = repIndexOf(file) ?? undefined;
+            if (opts.onlyUnparsed && !hasUnparsedVotes(prior.rep_judgments?.find(panel => panel.repetition === (rep ?? 0))))
+                continue;
+            savedExecution({ runDir, scenario: specById.get(id), prior,
+                ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version, }, mode, rep, readFileSync(join(runDir, file), "utf8"));
+        }
+    }
+    const references = Object.fromEntries(targets.map(id => [id, resolveJudgeReferences(specById.get(id), specDir)]));
     const targetSet = new Set(targets);
     const scenarioResults = [];
     for (const rec of recorded) {
@@ -227,7 +279,9 @@ export async function regradeRun(opts) {
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
             runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
+            references: references[id],
             expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
+            ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version,
         });
         const carry = overrides.get(id);
         if (prev?.schema === 3)
@@ -270,7 +324,7 @@ export async function regradeRun(opts) {
         // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
         // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
-        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets),
+        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets, opts.specDir, references),
         source_hash_roots: prev?.source_hash_roots,
         scenarios: scenarioResults,
     }, ctx);

package/dist/results.d.ts

@@ -105,6 +105,13 @@ export interface Judgment {
     judgeFormat?: "json" | "prose";
     /** One validation-guided retry after an invalid structured reply. */
     judgeRetries?: 1;
+    /** Content-addressed private input artifact; preserves exact prompts and execution coverage. */
+    input?: {
+        artifact: string;
+        sha256: string;
+        execution: "verified" | "unknown";
+        trace_sha256?: string;
+    };
 }
 export interface AdjudicationResult {
     /** The saved transcript repetition judged by every panel member (first rep by policy). */

package/dist/results.js

@@ -743,6 +743,11 @@ export function findTranscriptFile(runDir, scenarioId) {
  * for that claim is the code the model wrote.
  */
 export function preserveTranscript(resultsRoot, runDir, scenarioId) {
+    const record = existsSync(resultsPath(runDir)) ? readResults(runDir).scenarios.find(scenario => scenario.id === scenarioId) : undefined;
+    const judgeInputs = (record?.rep_judgments ?? []).flatMap(panel => panel.judgments.flatMap(judgment => {
+        const input = judgment.input;
+        return input && input.artifact === `judge-input.${input.sha256}.txt` && /^[a-f0-9]{64}$/.test(input.sha256) && existsSync(join(runDir, input.artifact)) ? [input.artifact] : [];
+    }));
     const files = [
         ...findTranscriptFiles(runDir, scenarioId),
         ...findJudgeRawFiles(runDir, scenarioId),
@@ -752,6 +757,7 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         // override whose justification was gitignored, and left `regate` with nothing
         // to re-evaluate on the one cell a human had disputed.
         ...findTraceFiles(runDir, scenarioId),
+        ...judgeInputs,
     ];
     if (files.length === 0)
         return;

package/dist/run.js

@@ -1,13 +1,15 @@
 import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { sourceHashes, sourceHashRoots } from "./sources.js";
-import { judgeResemblesSubject } from "./grade.js";
+import { judgeResemblesSubject, resolveJudgeReferences } from "./grade.js";
+import { inspectEvalAuthoring } from "./authoring-preflight.js";
 import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { evaluateOutputGates, hasOutputGates, runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { executionFacts, transcriptResponses } from "./judge-evidence.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
@@ -63,6 +65,11 @@ export async function runSkillModel(opts) {
         scenarios = spec.scenarios.filter((s) => wanted.has(s.id));
         log(`  --only ${opts.only.join(",")} — partial run, will not be ship-graded`);
     }
+    const authoringErrors = inspectEvalAuthoring({ ...spec, scenarios }).filter(f => f.severity === "error");
+    if (authoringErrors.length)
+        throw new Error(authoringErrors.map(f => `${f.scenario}: ${f.message}`).join("\n"));
+    const judgeReferences = Object.fromEntries(scenarios.map(s => [s.id, resolveJudgeReferences(s, dirname(opts.specPath))]));
+    const hashes = sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona, judgeReferences });
     if (judgeResemblesSubject(judge, model)) {
         log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
@@ -132,7 +139,7 @@ export async function runSkillModel(opts) {
             const rep = k;
             const total = repCounts[si];
             owners.push(si);
-            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions }));
+            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions, judgeReferences }));
         }
     });
     const flat = await runPool(tasks, opts.concurrency ?? 1);
@@ -145,7 +152,6 @@ export async function runSkillModel(opts) {
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
-    const hashes = sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona });
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
@@ -332,6 +338,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                     else {
                         const req = {
                             skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                            tools: scenario.tools,
                             // resolved like fixtures: relative to the spec's dir
                             systemPromptFile: scenario.systemPromptFile
                                 ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
@@ -461,23 +468,26 @@ async function runRep(scenario, rep, repCount, ctx) {
         // timeout that caused it, and it is that text, not the real reason, that would
         // reach the results record. The verdict is ERROR either way.
         let objective;
+        if (traces.length > 0)
+            writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+        const execution = executionFacts(traces, {
+            scenarioId: scenario.id, mode, repetition: rep, turns: scenario.turns.length,
+            subject: ctx.model, responses: transcriptResponses(transcript, scenario),
+        }, { source: "live" });
         if ((scenario.traceAssert || hasOutputGates(scenario)) && !adapterFailure && !noResponse && !infrastructureFailure) {
             const assertionResults = [...outputGate.assertions];
             let status = outputGate.status;
             let traceMeta = {};
             const outputMeta = outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {};
             if (scenario.traceAssert) {
-                if (traces.length > 0) {
-                    writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
-                }
-                const merged = mergeTraces(traces);
+                const merged = execution.status === "invalid" ? null : mergeTraces(traces);
                 if (unobservablePaths) {
                     status = "ERROR";
                     assertionResults.push({ kind: "unchanged_path", status: "ERROR", detail: "workspace changes could not be observed" });
                 }
                 else if (merged === null) {
                     status = "ERROR";
-                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: "no execution trace was produced" });
+                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: execution.status === "invalid" ? execution.reason : "no execution trace was produced" });
                 }
                 else {
                     const gate = evaluateTraceGates(scenario.traceAssert, merged);
@@ -534,6 +544,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             const o = await judgeOneRep({
                 runDir, spec, scenario, transcript, adapter: ctx.adapter, judge,
                 specDir: dirname(ctx.specPath), mode, rep: repCount > 1 ? rep : undefined, now,
+                execution, objective, references: ctx.judgeReferences[scenario.id],
             });
             verdict = o.verdict;
             reason = o.reason;
@@ -542,7 +553,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             judgeCalls = o.metrics?.judge_calls ?? 0;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        const subject = mergeTraces(traces)?.metrics;
+        const subject = execution.status === "invalid" ? undefined : mergeTraces(traces)?.metrics;
         return {
             verdict, reason, suspect, objective, judgment,
             metrics: {

package/dist/seeded.js

@@ -234,6 +234,7 @@ export async function runSeeded(scenario, opts) {
         model: opts.model,
         mode: opts.mode,
         turns: scenario.turns,
+        tools: scenario.tools,
         cwd: repo,
         // Resolved against the spec dir, exactly like fixtures and post-tests —
         // the arm's extensions (already absolute) join alongside, same as the

package/dist/sources.d.ts

@@ -1,4 +1,5 @@
 import type { Scenario } from "./spec.js";
+import { type JudgeReference } from "./judge-references.js";
 export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
 /**
  * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
@@ -103,7 +104,7 @@ export declare function fileSha256(path: string): string | null;
  */
 export declare function dirSha256(dir: string): string | null;
 export declare function stimulusDigest(s: Scenario): string;
-export declare function rubricDigest(s: Scenario): string;
+export declare function rubricDigest(s: Scenario, specDir?: string, references?: JudgeReference[]): string;
 export declare function policyDigest(s: Scenario): string;
 /** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export declare function gatesDigest(s: Scenario): string | null;
@@ -143,6 +144,8 @@ export interface SourceContext {
      * caller instead of leaving one quietly un-checked.
      */
     judgePersona: string;
+    /** Snapshot resolved before execution, shared by prompt construction and recorded rubric hashes. */
+    judgeReferences?: Record<string, JudgeReference[]>;
 }
 /**
  * Hash every source this run measures: SKILL.md, each distinct