@skill-harness/core 0.29.0 → 0.31.0

package/dist/regrade.js

@@ -1,11 +1,16 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { buildJudgePrompt, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
+import { createHash } from "node:crypto";
+import { parseModelRef } from "./adapters/types.js";
+import { prepareJudgePrompt, resolveJudgeReferences, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, tracePath, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
 import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
+import { loadExecutionFacts, transcriptResponses } from "./judge-evidence.js";
+import { withResourceMonitoring } from "./resource-monitor.js";
+import { throwIfProcessAborted } from "./util/process.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -18,7 +23,7 @@ import { providerFailureFromTranscript, executionFailureFromTranscript } from ".
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export function refreshRubricHashes(recorded, spec, judgedIds) {
+export function refreshRubricHashes(recorded, spec, judgedIds, specDir, references, refreshPersona = true) {
     if (!recorded)
         return undefined;
     const next = { ...recorded };
@@ -28,12 +33,27 @@ export function refreshRubricHashes(recorded, spec, judgedIds) {
         // Only refresh a key the run already carried: adding one for a scenario whose
         // rubric was never hashed would fabricate coverage.
         if (s && RUBRIC_PREFIX + id in next)
-            next[RUBRIC_PREFIX + id] = rubricDigest(s);
+            next[RUBRIC_PREFIX + id] = rubricDigest(s, specDir, references?.[id]);
     }
-    if (PERSONA_KEY in next)
+    if (refreshPersona && PERSONA_KEY in next)
         next[PERSONA_KEY] = personaDigest(spec.judge_persona);
     return next;
 }
+function savedExecution(opts, mode, rep, transcript) {
+    const repetition = rep ?? 0;
+    const panel = opts.prior?.rep_judgments?.find(panel => panel.repetition === repetition);
+    const hashes = [panel?.judgments[0]?.input?.trace_sha256, panel?.objective?.trace_sha256, opts.prior?.objective?.rep_trace_sha256?.[repetition],
+        (opts.prior?.reps ?? 1) === 1 ? opts.prior?.objective?.trace_sha256 : undefined].filter((value) => value !== undefined);
+    if (new Set(hashes).size > 1)
+        throw new Error(`${opts.scenario.id}#${repetition}: conflicting recorded trace digests; regrade refused.`);
+    const execution = loadExecutionFacts(tracePath(opts.runDir, opts.scenario.id, mode, rep), {
+        scenarioId: opts.scenario.id, mode, repetition, turns: opts.scenario.turns.length,
+        subject: opts.subject, piVersion: opts.piVersion, responses: transcriptResponses(transcript, opts.scenario),
+    }, hashes[0]);
+    if (execution.status === "invalid")
+        throw new Error(`${opts.scenario.id}#${repetition}: ${execution.reason} Regrade refused; retained results were not replaced.`);
+    return execution;
+}
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
@@ -56,12 +76,30 @@ export async function judgeOneRep(opts) {
             metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 },
         };
     }
-    const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
+    if (opts.execution?.status === "invalid") {
+        const reason = `execution evidence: ${opts.execution.reason}`;
+        appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
+        return { verdict: "ERROR", reason, suspect: false, objective: opts.objective,
+            metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 } };
+    }
+    const prompt = prepareJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript, execution: opts.execution, references: opts.references }, specDir);
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
+    const inputBytes = JSON.stringify({ schema: 1, judge, execution: opts.execution ?? null, prompts: g.prompts }) + "\n";
+    const inputHash = createHash("sha256").update(inputBytes).digest("hex");
+    const artifact = `judge-input.${inputHash}.txt`;
+    const inputPath = join(runDir, artifact);
+    if (existsSync(inputPath)) {
+        if (readFileSync(inputPath, "utf8") !== inputBytes)
+            throw new Error("judge input artifact has conflicting bytes");
+    }
+    else
+        writeFileSync(inputPath, inputBytes, { encoding: "utf8", flag: "wx", mode: 0o600 });
+    const input = { artifact, sha256: inputHash, execution: opts.execution?.status === "verified" ? "verified" : "unknown",
+        ...(opts.execution?.traceSha256 ? { trace_sha256: opts.execution.traceSha256 } : {}) };
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), formatJudgeRawReplies(g.rawReplies), "utf8");
     const outcome = normalizeRepOutcome({
         verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, input, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),
@@ -81,6 +119,7 @@ export async function judgeOneRep(opts) {
  * persists).
  */
 export async function regradeScenario(opts) {
+    const references = opts.references ?? resolveJudgeReferences(opts.scenario, opts.specDir);
     const now = opts.now ?? (() => new Date().toISOString());
     const mode = opts.mode ?? "green";
     let files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
@@ -109,10 +148,12 @@ export async function regradeScenario(opts) {
         const file = files.find(file => (repIndexOf(file) ?? 0) === repetition);
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
+        const execution = savedExecution(opts, mode, rep, transcript);
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
             objective: panel?.objective ?? (expected === 1 ? opts.prior?.objective : undefined),
+            execution, references,
         }));
     }
     const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
@@ -138,6 +179,7 @@ function hasUnparsedVotes(panel) {
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
 export async function regradeRun(opts) {
+    throwIfProcessAborted();
     const { runDir, spec, adapter, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
@@ -169,15 +211,23 @@ export async function regradeRun(opts) {
     const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
     const recorded = prev?.scenarios ?? spec.scenarios.map((s) => ({ id: s.id }));
     let targets = recorded.map((s) => s.id);
+    if (opts.only !== undefined) {
+        if (!opts.only.length || opts.only.some(id => !id.trim()) || new Set(opts.only).size !== opts.only.length)
+            throw new Error("Selected scenarios must be a nonempty unique ID list");
+        const unknown = opts.only.filter(id => !targets.includes(id));
+        if (unknown.length)
+            throw new Error(`Unknown recorded scenario(s): ${unknown.join(", ")}`);
+        targets = targets.filter(id => opts.only.includes(id));
+    }
     if (opts.onlySuspect) {
         if (!prev)
             throw new Error(`--suspect-only needs a prior results.yaml in ${runDir}`);
-        targets = prev.scenarios
+        targets = prev.scenarios.filter(s => targets.includes(s.id))
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
     }
     if (opts.onlyUnparsed)
-        targets = recorded.filter(s => s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
+        targets = recorded.filter(s => targets.includes(s.id) && s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
     // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
     // cannot turn an undelivered or unauthenticated transcript into product evidence.
     if (prev?.schema === 3) {
@@ -188,6 +238,8 @@ export async function regradeRun(opts) {
         // Nothing judgeable — a no-op, preserving both evidence and judge identity.
         return prev;
     }
+    if (prev && targets.length !== recorded.length && (judge.provider !== prev.judge.provider || judge.model !== prev.judge.model))
+        throw new Error("Selective regrade must use the recorded judge; regrade the whole run to change judge");
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
         const files = findTranscriptFiles(runDir, record.id, mode);
@@ -214,73 +266,92 @@ export async function regradeRun(opts) {
         if (drifted.length)
             throw new Error(`criterion count changed for ${drifted.join(", ")} — use full grade instead of --unparsed-only`);
     }
-    const targetSet = new Set(targets);
-    const scenarioResults = [];
-    for (const rec of recorded) {
-        const id = rec.id;
-        if (!targetSet.has(id)) {
-            scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
-            continue;
+    // Refuse the whole selected batch before spending on any earlier cell. Recheck
+    // again immediately before each call so a later filesystem edit is not trusted.
+    for (const id of targets) {
+        const prior = recordedById.get(id);
+        for (const file of findTranscriptFiles(runDir, id, mode)) {
+            const rep = repIndexOf(file) ?? undefined;
+            if (opts.onlyUnparsed && !hasUnparsedVotes(prior.rep_judgments?.find(panel => panel.repetition === (rep ?? 0))))
+                continue;
+            savedExecution({ runDir, scenario: specById.get(id), prior,
+                ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version, }, mode, rep, readFileSync(join(runDir, file), "utf8"));
         }
-        const scenario = specById.get(id); // guaranteed present by the guard above
-        const prevScenario = prev?.scenarios.find((s) => s.id === id);
-        const threshold = effectiveThreshold(prevScenario, scenario);
-        const rr = await regradeScenario({
-            runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
-            expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
-        });
-        const carry = overrides.get(id);
-        if (prev?.schema === 3)
-            rr.criterion_count = scenario.checklist.length;
-        rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
-        rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
-        // Gates still describe the original subject execution. An adjudication of
-        // an untouched repetition also survives, bounded by rebuild's conflict guards.
-        const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
-        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
-    }
-    const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
-    const results = writeResults(runDir, {
-        schema: prev?.schema,
-        subject_invocations: prev?.subject_invocations,
-        skill: spec.skill,
-        harness: prev?.harness ?? "pi",
-        // The harness CLI that produced these transcripts, carried verbatim: a re-grade
-        // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
-        // here would credit the old transcripts to a version that never ran them.
-        harness_cli_version: prev?.harness_cli_version,
-        delivery_canary: prev?.delivery_canary,
-        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
-        // only record that a `+<arm>` run actually delegated: rebuilding the draft
-        // field-by-field without it silently deleted `definitions`/`ledger_events`
-        // from any arm run that was ever re-graded, leaving a record
-        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
-        // `delivery_canary` and `source_hashes` are carried here.
-        arm: prev?.arm,
-        model: prev?.model ?? "unknown",
-        judge: { provider: judge.provider, model: judge.model },
-        timestamp: prev?.timestamp ?? now(),
-        label: prev?.label ?? null,
-        mode,
-        // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
-        // the recorded **stimulus** hashes stay, keeping an honestly-stale run honestly
-        // stale. The rubric hashes are a different matter: this re-grade applied the
-        // CURRENT checklist and persona to those transcripts, so "the verdicts reflect
-        // today's rubric" is now a true statement about the record, and the hashes should
-        // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
-        // *stimulus* hashes stay" — see refreshRubricHashes.
-        partial: prev?.partial,
-        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets),
-        source_hash_roots: prev?.source_hash_roots,
-        scenarios: scenarioResults,
-    }, ctx);
-    const g = results.effective_grade;
-    if (ctx) {
-        appendJournal(runDir, {
-            event: "score", ts: now(),
-            passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note,
-        });
     }
-    return results;
+    const references = Object.fromEntries(targets.map(id => [id, resolveJudgeReferences(specById.get(id), specDir)]));
+    return withResourceMonitoring(runDir, async () => {
+        const targetSet = new Set(targets);
+        const scenarioResults = [];
+        for (const rec of recorded) {
+            throwIfProcessAborted();
+            const id = rec.id;
+            if (!targetSet.has(id)) {
+                scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
+                continue;
+            }
+            const scenario = specById.get(id); // guaranteed present by the guard above
+            const prevScenario = prev?.scenarios.find((s) => s.id === id);
+            const threshold = effectiveThreshold(prevScenario, scenario);
+            const rr = await regradeScenario({
+                runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
+                references: references[id],
+                expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
+                ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version,
+            });
+            const carry = overrides.get(id);
+            if (prev?.schema === 3)
+                rr.criterion_count = scenario.checklist.length;
+            rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
+            rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
+            // Gates still describe the original subject execution. An adjudication of
+            // an untouched repetition also survives, bounded by rebuild's conflict guards.
+            const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
+            scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
+        }
+        const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
+        const results = writeResults(runDir, {
+            schema: prev?.schema,
+            subject_invocations: prev?.subject_invocations,
+            skill: spec.skill,
+            harness: prev?.harness ?? "pi",
+            // The harness CLI that produced these transcripts, carried verbatim: a re-grade
+            // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
+            // here would credit the old transcripts to a version that never ran them.
+            harness_cli_version: prev?.harness_cli_version,
+            delivery_canary: prev?.delivery_canary,
+            delivery_observations: prev?.delivery_observations,
+            // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+            // only record that a `+<arm>` run actually delegated: rebuilding the draft
+            // field-by-field without it silently deleted `definitions`/`ledger_events`
+            // from any arm run that was ever re-graded, leaving a record
+            // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+            // `delivery_canary` and `source_hashes` are carried here.
+            arm: prev?.arm,
+            model: prev?.model ?? "unknown",
+            judge: { provider: judge.provider, model: judge.model },
+            timestamp: prev?.timestamp ?? now(),
+            label: prev?.label ?? null,
+            mode,
+            // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
+            // the recorded **stimulus** hashes stay, keeping an honestly-stale run honestly
+            // stale. The rubric hashes are a different matter: this re-grade applied the
+            // CURRENT checklist and persona to those transcripts, so "the verdicts reflect
+            // today's rubric" is now a true statement about the record, and the hashes should
+            // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
+            // *stimulus* hashes stay" — see refreshRubricHashes.
+            partial: prev?.partial,
+            source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets, opts.specDir, references, targets.length === recorded.length),
+            source_hash_roots: prev?.source_hash_roots,
+            scenarios: scenarioResults,
+        }, ctx);
+        const g = results.effective_grade;
+        if (ctx) {
+            appendJournal(runDir, {
+                event: "score", ts: now(),
+                passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note,
+            });
+        }
+        return results;
+    });
 }
 //# sourceMappingURL=regrade.js.map

package/dist/rescore.js

@@ -95,6 +95,7 @@ export function rescoreRun(opts) {
         // threshold to reps the recorded harness already produced.
         harness_cli_version: prev.harness_cli_version,
         delivery_canary: prev.delivery_canary,
+        delivery_observations: prev.delivery_observations,
         // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
         // only record that a `+<arm>` run actually delegated: rebuilding the draft
         // field-by-field without it silently deleted `definitions`/`ledger_events`

package/dist/resource-monitor.d.ts

@@ -0,0 +1,3 @@
+import { type ProcessScope } from "./util/process.js";
+/** Per-operation metadata only, with bounded retention and low-frequency atomic snapshots. */
+export declare function withResourceMonitoring<T>(directory: string, work: () => Promise<T>, options?: ProcessScope): Promise<T>;

package/dist/resource-monitor.js

@@ -0,0 +1,72 @@
+import { mkdirSync, writeFileSync, renameSync } from "node:fs";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { getProcessSignal, withProcessScope } from "./util/process.js";
+/** Per-operation metadata only, with bounded retention and low-frequency atomic snapshots. */
+export async function withResourceMonitoring(directory, work, options = {}) {
+    mkdirSync(directory, { recursive: true, mode: 0o700 });
+    const id = randomUUID();
+    const path = join(directory, `resources-${id}.json`), pending = path + ".tmp";
+    const controller = new AbortController();
+    const inherited = options.signal ?? getProcessSignal();
+    const abort = () => controller.abort(inherited?.reason);
+    if (inherited?.aborted)
+        abort();
+    inherited?.addEventListener("abort", abort, { once: true });
+    const records = [];
+    const active = new Map();
+    let total = 0, dropped = 0, droppedActive = 0, state = "running", flushError;
+    const started = new Date().toISOString();
+    const flush = () => {
+        try {
+            writeFileSync(pending, JSON.stringify({ schema: 1, scope: "harness-owned-processes", started, updated: new Date().toISOString(), state,
+                hostHealth: "unmeasured", sampling: "one-second-owned-tree", totalProcesses: total, droppedObservations: dropped, droppedActiveObservations: droppedActive, active: [...active.values()], observations: records }, null, 2) + "\n", { mode: 0o600 });
+            renameSync(pending, path);
+        }
+        catch (error) {
+            flushError = error;
+            controller.abort(error);
+        }
+    };
+    flush();
+    if (flushError) {
+        inherited?.removeEventListener("abort", abort);
+        throw flushError;
+    }
+    const timer = setInterval(flush, 5000);
+    timer.unref();
+    try {
+        return await withProcessScope({ ...options, signal: controller.signal, onSample: record => {
+                if (record.pid !== null && (active.has(record.pid) || active.size < 256))
+                    active.set(record.pid, record);
+                else
+                    droppedActive++;
+                options.onSample?.(record);
+            }, onObservation: record => {
+                total++;
+                if (record.pid !== null)
+                    active.delete(record.pid);
+                if (records.length < 256)
+                    records.push(record);
+                else
+                    dropped++;
+                if (record.reason === "resource-limit" || record.reason === "output-limit")
+                    controller.abort(new Error(record.reason));
+                options.onObservation?.(record);
+            } }, work);
+    }
+    catch (error) {
+        state = "interrupted";
+        throw error;
+    }
+    finally {
+        clearInterval(timer);
+        inherited?.removeEventListener("abort", abort);
+        if (state === "running")
+            state = "settled";
+        flush();
+        if (flushError)
+            throw new Error(`Resource summary could not be retained: ${String(flushError)}`);
+    }
+}
+//# sourceMappingURL=resource-monitor.js.map

package/dist/results.d.ts

@@ -3,6 +3,7 @@ import { type ScenarioVerdict } from "./score.js";
 import type { Verdict } from "./score.js";
 import type { ShipBar, Scenario } from "./spec.js";
 import type { CostSource } from "./capture-trace-types.js";
+import type { CanaryObservations } from "./canary.js";
 export interface SubjectUsage {
     repetition: number;
     inputTokens: number | null;
@@ -105,6 +106,13 @@ export interface Judgment {
     judgeFormat?: "json" | "prose";
     /** One validation-guided retry after an invalid structured reply. */
     judgeRetries?: 1;
+    /** Content-addressed private input artifact; preserves exact prompts and execution coverage. */
+    input?: {
+        artifact: string;
+        sha256: string;
+        execution: "verified" | "unknown";
+        trace_sha256?: string;
+    };
 }
 export interface AdjudicationResult {
     /** The saved transcript repetition judged by every panel member (first rep by policy). */
@@ -174,10 +182,10 @@ export interface ResultsFile {
      */
     harness_cli_version?: string;
     /**
-     * `pass` when this run proved, before spending the wave, that the skill body was
-     * reachable in the model's context (see canary.ts). `skipped` when the probe was
-     * asked for but could not be performed — SKILL.md has no `## ` heading to quote
-     * back, so no reply could prove anything. Absent means the probe was not asked
+     * Compatibility status for the optional instructed loading/quote probe.
+     * See delivery_observations for what was actually observed; a quote by itself
+     * does not establish loading, and the probe never tests autonomous selection.
+     * `skipped` means the skill has no `## ` heading. Absent means the probe was not asked
      * for — never that it failed, because a failed canary aborts the run and no
      * results.yaml is written.
      *
@@ -192,6 +200,10 @@ export interface ResultsFile {
      * gitignored, and this claim has to survive a commit.
      */
     delivery_canary?: "pass" | "skipped";
+    /** Optional for historical records; never inferred from a legacy pass alone. */
+    delivery_observations?: CanaryObservations & {
+        probeCalls: 0 | 1;
+    };
     skill: string;
     harness: string;
     model: string;

package/dist/results.js

@@ -224,6 +224,7 @@ export function finalizeResults(draft, ctx) {
         // not report a version must not look like one that reported "nothing".
         ...(draft.harness_cli_version ? { harness_cli_version: draft.harness_cli_version } : {}),
         ...(draft.delivery_canary ? { delivery_canary: draft.delivery_canary } : {}),
+        ...(draft.delivery_observations ? { delivery_observations: draft.delivery_observations } : {}),
         skill: draft.skill,
         harness: draft.harness,
         model: draft.model,
@@ -498,6 +499,9 @@ export function applyOverride(results, scenarioId, override, note) {
 const GITIGNORE_BODY = `# skill-harness: commit verdicts (results.yaml), ignore generated artifacts.
 *.txt
 *.jsonl
+resources-*.json
+resources-*.json.tmp
+reuse-source-*.json
 report.html
 !results.yaml
 `;
@@ -743,6 +747,11 @@ export function findTranscriptFile(runDir, scenarioId) {
  * for that claim is the code the model wrote.
  */
 export function preserveTranscript(resultsRoot, runDir, scenarioId) {
+    const record = existsSync(resultsPath(runDir)) ? readResults(runDir).scenarios.find(scenario => scenario.id === scenarioId) : undefined;
+    const judgeInputs = (record?.rep_judgments ?? []).flatMap(panel => panel.judgments.flatMap(judgment => {
+        const input = judgment.input;
+        return input && input.artifact === `judge-input.${input.sha256}.txt` && /^[a-f0-9]{64}$/.test(input.sha256) && existsSync(join(runDir, input.artifact)) ? [input.artifact] : [];
+    }));
     const files = [
         ...findTranscriptFiles(runDir, scenarioId),
         ...findJudgeRawFiles(runDir, scenarioId),
@@ -752,6 +761,7 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         // override whose justification was gitignored, and left `regate` with nothing
         // to re-evaluate on the one cell a human had disputed.
         ...findTraceFiles(runDir, scenarioId),
+        ...judgeInputs,
     ];
     if (files.length === 0)
         return;

package/dist/run.d.ts

@@ -31,8 +31,8 @@ export interface RunOptions {
      */
     only?: string[];
     /**
-     * Green mode only: spend ONE probe up front proving the skill reaches the model,
-     * and abort the run if it doesn't (see canary.ts). Off by default — it costs a
+     * Green mode only: spend ONE instructed loading/read/quote probe up front,
+     * and stop the wave if inconclusive (see canary.ts). Off by default — it costs a
      * rep, and the deterministic half of this failure class (a skill dir that isn't
      * there) is already refused by the adapter for free.
      */