@skill-harness/core 0.30.0 → 0.31.0

package/dist/authoring-preflight.d.ts

@@ -1,7 +1,7 @@
 import type { Spec } from "./spec.js";
 export interface AuthoringFinding {
     scenario: string;
-    code: "INPUT_DISCOVERY" | "READ_COUNT" | "PATH_PORTABILITY" | "COMMAND_SPELLING" | "TEST_SYNTAX" | "CITATION_FORMAT" | "UNAVAILABLE_FORBIDDEN_TOOL" | "UNAVAILABLE_REQUIRED_TOOL";
+    code: "INPUT_DISCOVERY" | "READ_COUNT" | "PATH_PORTABILITY" | "COMMAND_SPELLING" | "TEST_SYNTAX" | "CITATION_FORMAT" | "UNAVAILABLE_FORBIDDEN_TOOL" | "UNAVAILABLE_REQUIRED_TOOL" | "JUDGE_REFERENCE_GAP" | "MISSING_COVERAGE_MAP";
     severity?: "error";
     message: string;
 }
@@ -9,6 +9,14 @@ export interface AuthoringFinding {
 export declare function inspectEvalAuthoring(spec: Spec, profile?: {
     tools?: readonly string[];
     fixtureFiles?: (scenario: Spec["scenarios"][number]) => readonly string[];
+    /** Explicitly selected fixture/reference text only. No host search or automatic judge disclosure. */
+    fixtureText?: (scenario: Spec["scenarios"][number], file: string) => string | null;
+    referenceText?: (file: string) => string | null;
 }): AuthoringFinding[];
 /** Names only, never file contents. Do not follow symlinks or inspect private host directories. */
 export declare function fixtureInventory(root: string): string[];
+/** Optional text inventory for offline lint. Bounded; never expands the judge's input. */
+export declare function authoringEvidenceProfile(specDir: string): {
+    fixtureText: (s: Spec["scenarios"][number], file: string) => string | null;
+    referenceText: (file: string) => string | null;
+};

package/dist/authoring-preflight.js

@@ -1,5 +1,5 @@
-import { readdirSync } from "node:fs";
-import { join } from "node:path";
+import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { join, relative, isAbsolute, resolve } from "node:path";
 import { MARKERS } from "./workspace.js";
 /** Review hints only. A valid exact-format contract may intentionally trigger one. */
 export function inspectEvalAuthoring(spec, profile = {}) {
@@ -7,6 +7,21 @@ export function inspectEvalAuthoring(spec, profile = {}) {
     for (const s of spec.scenarios) {
         const tools = profile.tools ?? s.tools;
         const add = (code, message) => findings.push({ scenario: s.id, code, message });
+        if (!s.covers?.length)
+            add("MISSING_COVERAGE_MAP", "No instruction-section coverage is declared. Map the behavior this scenario can detect; a map is not proof of coverage.");
+        if (s.judgeEvidence === "response" && profile.fixtureText && profile.referenceText) {
+            const references = (s.judgeReferences ?? []).map(profile.referenceText).filter((text) => text !== null);
+            const normalize = (text) => text.replace(/\r\n/g, "\n").trim();
+            const visible = [...references, s.judgeContext ?? ""].map(normalize);
+            for (const file of profile.fixtureFiles?.(s) ?? []) {
+                if (!/\.(?:md|txt|json|ya?ml|csv)$/i.test(file))
+                    continue;
+                const source = profile.fixtureText(s, file);
+                if (source !== null && normalize(source) && !visible.some(ref => ref.includes(normalize(source)))) {
+                    add("JUDGE_REFERENCE_GAP", `Fixture ${file} is not present verbatim in judge references/context. Verify whether its claims are needed to judge valid answers, including brand, goals and conflicting facts. Deliberately private sources may remain excluded; document the reason. This is an inventory warning, not a semantic verdict.`);
+                }
+            }
+        }
         for (const call of s.traceAssert?.require_calls ?? []) {
             const pattern = call.args?.path?.matches;
             if (tools && !tools.includes(call.tool)) {
@@ -41,13 +56,18 @@ export function inspectEvalAuthoring(spec, profile = {}) {
 /** Names only, never file contents. Do not follow symlinks or inspect private host directories. */
 export function fixtureInventory(root) {
     const files = [];
-    const visit = (directory, prefix) => {
+    let entries = 0;
+    const visit = (directory, prefix, depth = 0) => {
+        if (depth > 32)
+            throw new Error("Fixture inventory exceeds 32 directory levels; select a smaller fixture");
         for (const entry of readdirSync(directory, { withFileTypes: true })) {
+            if (++entries > 4096)
+                throw new Error("Fixture inventory exceeds 4096 entries; select a smaller fixture");
             if (entry.name === ".git" || entry.isSymbolicLink())
                 continue;
             const name = prefix + entry.name;
             if (entry.isDirectory())
-                visit(join(directory, entry.name), name + "/");
+                visit(join(directory, entry.name), name + "/", depth + 1);
             else if (entry.isFile())
                 files.push(MARKERS.some(marker => name.startsWith(marker + "/")) ? name.slice(name.indexOf("/") + 1) : name);
         }
@@ -55,4 +75,37 @@ export function fixtureInventory(root) {
     visit(root, "");
     return [...new Set(files)].sort();
 }
+/** Optional text inventory for offline lint. Bounded; never expands the judge's input. */
+export function authoringEvidenceProfile(specDir) {
+    let remaining = 2 * 1024 * 1024;
+    const cache = new Map();
+    const text = (file) => {
+        const path = resolve(specDir, file);
+        if (cache.has(path))
+            return cache.get(path);
+        let value = null;
+        try {
+            const rel = relative(realpathSync(specDir), realpathSync(path));
+            const stat = statSync(path), size = stat.size;
+            if (!stat.isFile() || isAbsolute(rel) || rel === ".." || rel.startsWith("../") || rel.startsWith("..\\") || size > 65536 || size > remaining)
+                return null;
+            remaining -= size;
+            value = new TextDecoder("utf-8", { fatal: true }).decode(readFileSync(path));
+        }
+        catch { /* Unreadable/binary source is not invented as empty evidence. */ }
+        cache.set(path, value);
+        return value;
+    };
+    const fixture = (s) => typeof s.workspace === "object" ? s.workspace.fixture : s.fixture;
+    return {
+        fixtureText: (s, file) => {
+            const root = fixture(s);
+            if (!root)
+                return null;
+            // Overlay order is the same as workspace creation: later pending content wins.
+            return text(join(root, "_uncommitted", file)) ?? text(join(root, "_staged", file)) ?? text(join(root, file));
+        },
+        referenceText: text,
+    };
+}
 //# sourceMappingURL=authoring-preflight.js.map

package/dist/authoring.js

@@ -300,7 +300,15 @@ function draftIssues(state, draft) {
                     issues.push(`${scenario.id} support file is missing: ${path}`);
             }
         }
-        for (const finding of inspectEvalAuthoring(spec, { fixtureFiles: scenario => {
+        for (const finding of inspectEvalAuthoring(spec, {
+            referenceText: path => lookup("evals", path),
+            fixtureText: (scenario, path) => {
+                const fixture = typeof scenario.workspace === "object" ? scenario.workspace.fixture : scenario.fixture;
+                if (!fixture)
+                    return null;
+                const prefix = fixture.replace(/\/$/, "");
+                return lookup("evals", `${prefix}/_uncommitted/${path}`) ?? lookup("evals", `${prefix}/_staged/${path}`) ?? lookup("evals", `${prefix}/${path}`);
+            }, fixtureFiles: scenario => {
                 const fixture = typeof scenario.workspace === "object" ? scenario.workspace.fixture : scenario.fixture;
                 if (!fixture)
                     return [];
@@ -310,7 +318,8 @@ function draftIssues(state, draft) {
                 const actual = resolve(locations(state).evals, fixture);
                 noLinks(actual);
                 return [...new Set([...staged, ...(existsSync(actual) && lstatSync(actual).isDirectory() ? fixtureInventory(actual) : [])])];
-            } })) {
+            }
+        })) {
             (finding.severity === "error" ? issues : warnings).push(`${finding.scenario}: ${finding.message}`);
         }
         if (state.answers.placement === "separate-repo" && existsSync(join(state.skillDir, "tests", "specification.yaml"))) {

package/dist/calibration.d.ts

@@ -37,7 +37,11 @@ export interface LoadedCalibration {
     spec: Spec;
     controls: Array<BoundControl & {
         prompt: string;
+        executionEvidence: "verified" | "unknown";
     }>;
+    /** Calibration judges supplied answers; even a verified trace is not a gate evaluation. */
+    calibrationScope: "answer-judge";
+    executionGates: "not-evaluated";
     ready: boolean;
     issues: string[];
 }
@@ -49,6 +53,43 @@ export declare function prepareCalibration(options: {
 }): LoadedCalibration;
 /** Offline validation only. No Pi, provider access, or result rewrite. */
 export declare function loadCalibration(manifestPath: string): LoadedCalibration;
+export interface CalibrationReuseSource {
+    directory: string;
+    manifestSha256: string;
+    artifacts: {
+        path: string;
+        sha256: string;
+    }[];
+}
+export interface CalibrationSelection {
+    only?: string[];
+    reuse?: string[];
+}
+export interface CalibrationPlan {
+    schema: 1;
+    purpose: "judge-calibration-plan";
+    manifestSha256: string;
+    judge: ModelRef;
+    calibrationScope: "answer-judge";
+    executionGates: "not-evaluated";
+    availableCases: number;
+    cases: number;
+    partial: boolean;
+    ready: boolean;
+    issues: string[];
+    notes: string[];
+    judgeCallsPlanned: number;
+    reused: number;
+    controls: {
+        id: string;
+        action: "judge" | "reuse";
+        source?: CalibrationReuseSource;
+    }[];
+}
+/** Preview eligible historical judgments. Reuse is not a fresh stability measurement. */
+export declare function planCalibration(loaded: LoadedCalibration, options: CalibrationSelection & {
+    judge: ModelRef;
+}): CalibrationPlan;
 export interface CalibrationSummary {
     schema: 1;
     purpose: "judge-calibration";
@@ -60,6 +101,20 @@ export interface CalibrationSummary {
     matched: number;
     judgeCalls: number;
     status: "matched" | "mismatch" | "error";
+    calibrationScope: "answer-judge";
+    executionGates: "not-evaluated";
+    availableCases: number;
+    partial: boolean;
+    disagreements: {
+        id: string;
+        expected: "PASS" | "FAIL";
+        observed: "PASS" | "FAIL";
+    }[];
+    reused: {
+        id: string;
+        source: CalibrationReuseSource;
+    }[];
+    /** Legacy direction aliases for label disagreements, not adjudicated judge errors. */
     falsePasses: string[];
     falseFailures: string[];
     errors: string[];
@@ -72,6 +127,8 @@ export declare function runCalibration(loaded: LoadedCalibration, options: {
     judge: ModelRef;
     output: string;
     allowMeteredJudge?: boolean;
+    only?: string[];
+    reuse?: string[];
     onProgress?: (progress: {
         id: string;
         index: number;

package/dist/calibration.js

@@ -2,11 +2,13 @@ import { createHash } from "node:crypto";
 import { mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { loadSpec } from "./spec.js";
-import { prepareJudgePrompt, judgeInWorkspace } from "./grade.js";
+import { prepareJudgePrompt, judgeInWorkspace, parseStructuredJudgeReply } from "./grade.js";
 import { loadExecutionFacts, transcriptResponses, unavailableExecution } from "./judge-evidence.js";
 import { assertJudgeAllowed } from "./judge-policy.js";
 import { rubricDigest } from "./sources.js";
 import { resolveJudgeReferences } from "./judge-references.js";
+import { withResourceMonitoring } from "./resource-monitor.js";
+import { isProcessControlError, throwIfProcessAborted } from "./util/process.js";
 const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
 const HASH = /^[a-f0-9]{64}$/;
 const ID = /^[A-Za-z][A-Za-z0-9_-]{0,79}$/;
@@ -88,11 +90,12 @@ function validateManifest(manifest, manifestPath, manifestSha256) {
             if (execution.status !== "verified" || execution.coverage !== "complete")
                 throw new Error(`${control.id}: execution evidence ERROR: ${execution.reason}`);
         }
-        else if (scenario.traceAssert) {
-            issues.push(`${control.id}: execution assertions need bound trace evidence`);
-        }
-        const prompt = prepareJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript: control.transcript, execution, references }, specDir);
-        return { ...control, prompt };
+        const scope = control.kind === "saved"
+            ? "This is a selected saved answer. Only the separately verified execution facts establish observed execution."
+            : "This is an authored answer control. Any supplied code, staged diff, test output or gate trailer is synthetic control material, not evidence that the subject executed it.";
+        const prompt = prepareJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript: control.transcript, execution, references }, specDir) +
+            `\nCALIBRATION SCOPE (trusted harness metadata):\n${scope} Assess the supplied response/code against the semantic checklist and references. Tool use, execution and unchanged-file claims remain unverified unless supported by the separate execution facts. This run does not evaluate objective execution gates, even when a trace is verified. Keep the required JSON response format.\n`;
+        return { ...control, prompt, executionEvidence: execution.status === "verified" ? "verified" : "unknown" };
     });
     for (const control of controls.filter(c => c.kind === "equivalent")) {
         const original = controls.find(c => c.id === control.equivalentTo);
@@ -105,7 +108,8 @@ function validateManifest(manifest, manifestPath, manifestSha256) {
         issues.push("Add a deliberately incorrect negative control");
     if (!controls.some(c => c.expected === "PASS" && c.kind === "equivalent"))
         issues.push("Add an equivalent correct answer to detect wording overfit");
-    return { manifestPath: resolve(manifestPath), manifestSha256, specDir, spec, controls, ready: !issues.length, issues };
+    return { manifestPath: resolve(manifestPath), manifestSha256, specDir, spec, controls,
+        calibrationScope: "answer-judge", executionGates: "not-evaluated", ready: !issues.length, issues };
 }
 /** Deterministic authoring helper. Hashes are produced by code, never requested from a model. */
 export function prepareCalibration(options) {
@@ -144,69 +148,205 @@ export function loadCalibration(manifestPath) {
         throw new Error("Calibration manifest exceeds 4 MiB");
     return validateManifest(JSON.parse(bytes.toString("utf8")), manifestPath, hash(bytes));
 }
-/** Explicit Pi-backed judging; semantic mismatches continue, infrastructure errors stop the remaining calls. */
-export async function runCalibration(loaded, options) {
+/** Explicitly selected prior evidence only: no directory discovery and no model calls. */
+function calibrationPlan(loaded, options) {
     const current = loadCalibration(loaded.manifestPath);
     if (current.manifestSha256 !== loaded.manifestSha256)
         throw new Error("Calibration changed after validation");
-    if (!current.ready)
-        throw new Error(`Calibration is not ready: ${current.issues.join("; ")}`);
+    if (options.only && (!options.only.length || new Set(options.only).size !== options.only.length ||
+        options.only.some(id => !current.controls.some(control => control.id === id))))
+        throw new Error("Select distinct existing calibration control IDs");
+    if ((options.reuse?.length ?? 0) > 20)
+        throw new Error("Select at most 20 prior calibration directories");
+    const selected = current.controls.filter(control => !options.only || options.only.includes(control.id));
+    const plan = {
+        schema: 1, purpose: "judge-calibration-plan", manifestSha256: current.manifestSha256, judge: options.judge,
+        calibrationScope: "answer-judge", executionGates: "not-evaluated", availableCases: current.controls.length,
+        cases: selected.length, partial: selected.length !== current.controls.length, ready: current.ready,
+        issues: [...current.issues], notes: [], judgeCallsPlanned: selected.length, reused: 0,
+        controls: selected.map(control => ({ id: control.id, action: "judge" })),
+    };
+    const retained = new Map();
+    // ModelRef currently has no sampling/settings fields. Unknown future settings must not silently reuse.
+    const sameJudge = (value) => value?.provider === options.judge.provider && value?.model === options.judge.model &&
+        Object.keys(value).sort().join(",") === "model,provider" && Object.keys(options.judge).sort().join(",") === "model,provider";
+    for (const directory of [...new Set(options.reuse ?? [])].map(path => resolve(path))) {
+        const evidence = [];
+        const files = new Map();
+        const read = (path, limit = 2_097_152) => {
+            const bytes = readFileSync(selectedFile(directory, path, limit));
+            evidence.push({ path, sha256: hash(bytes) });
+            files.set(path, bytes);
+            return JSON.parse(bytes.toString("utf8"));
+        };
+        const inputs = read("inputs.json", 33_554_432), summary = read("summary.json");
+        if (summary.schema !== 1 || summary.purpose !== "judge-calibration" || !HASH.test(summary.manifestSha256) ||
+            summary.manifestSha256 !== inputs.manifestSha256 || !Array.isArray(inputs.controls) || inputs.controls.length > 100 ||
+            !["matched", "mismatch", "error"].includes(summary.status) ||
+            !Array.isArray(summary.errors) || !Array.isArray(summary.unrun))
+            throw new Error(`Invalid prior calibration evidence: ${directory}`);
+        if (!sameJudge(inputs.judge) || !sameJudge(summary.judge)) {
+            plan.notes.push(`${directory}: different judge; no results reused`);
+            continue;
+        }
+        const common = evidence.slice();
+        for (const control of selected) {
+            const prior = inputs.controls.find((candidate) => candidate.id === control.id);
+            if (!prior || summary.errors.includes(control.id) || summary.unrun.includes(control.id))
+                continue;
+            if (prior.scenarioId !== control.scenarioId || prior.expected !== control.expected || prior.prompt !== control.prompt ||
+                prior.transcript !== control.transcript || prior.transcriptSha256 !== control.transcriptSha256 ||
+                prior.rubricSha256 !== control.rubricSha256 || prior.review?.status !== "accepted" || !prior.review.by?.trim())
+                continue;
+            evidence.length = 0;
+            evidence.push(...common);
+            files.clear();
+            const request = read(`${control.id}.request.json`), result = read(`${control.id}.result.json`);
+            if (request.prompt !== control.prompt || request.sha256 !== hash(control.prompt) || result.expected !== control.expected ||
+                result.judgeFormat !== "json" || typeof result.raw !== "string" || !Array.isArray(result.rawReplies) ||
+                result.rawReplies.length < 1 || result.rawReplies.length > 2 || !Array.isArray(result.prompts) ||
+                result.prompts.length !== result.rawReplies.length || result.prompts[0] !== control.prompt ||
+                result.raw !== result.rawReplies.at(-1))
+                throw new Error(`Invalid prior control evidence: ${control.id} in ${directory}`);
+            for (let index = 0; index < result.rawReplies.length; index++) {
+                const sent = read(`${control.id}.attempt-${index + 1}.request.json`);
+                const reply = read(`${control.id}.attempt-${index + 1}.reply.json`);
+                if (typeof result.rawReplies[index] !== "string" || typeof result.prompts[index] !== "string" ||
+                    sent.prompt !== result.prompts[index] || sent.sha256 !== hash(sent.prompt) ||
+                    reply.reply !== result.rawReplies[index] || reply.sha256 !== hash(reply.reply))
+                    throw new Error(`Prior request/reply changed: ${control.id} in ${directory}`);
+            }
+            const scenario = current.spec.scenarios.find(scenario => scenario.id === control.scenarioId);
+            if (result.prompts.length === 2) {
+                let validationError;
+                try {
+                    parseStructuredJudgeReply(result.rawReplies[0], scenario.checklist.length);
+                }
+                catch (error) {
+                    validationError = error.message;
+                }
+                // Bind the validation-guided retry as well as the initial request; arbitrary follow-ups are not equivalent.
+                const retry = `${control.prompt}\n\nYour reply was not valid: ${JSON.stringify(validationError)}; reply with only the JSON object.`;
+                if (validationError === undefined || /^\[judge error:/i.test(result.rawReplies[0].trim()) || result.prompts[1] !== retry) {
+                    throw new Error(`Prior retry prompt is not equivalent: ${control.id} in ${directory}`);
+                }
+            }
+            const parsed = parseStructuredJudgeReply(result.raw, scenario.checklist.length);
+            const consistent = parsed.verdict === "PASS" ? parsed.criteria.every(vote => vote.verdict === "PASS") : parsed.criteria.some(vote => vote.verdict === "FAIL");
+            if (result.suspect || !consistent || parsed.verdict !== result.verdict || parsed.reason !== result.reason ||
+                JSON.stringify(parsed.criteria) !== JSON.stringify(result.criteria))
+                throw new Error(`Invalid prior judgment: ${control.id} in ${directory}`);
+            if (retained.has(control.id) && retained.get(control.id).result.verdict !== result.verdict) {
+                plan.issues.push(`${control.id}: prior judgments disagree; inspect them or select a fresh run explicitly`);
+                continue;
+            }
+            if (!retained.has(control.id)) {
+                // Freeze the exact selected records now; later writes cannot substitute changed source bytes.
+                retained.set(control.id, { result, files: new Map(files) });
+                const row = plan.controls.find(row => row.id === control.id);
+                row.action = "reuse";
+                row.source = { directory, manifestSha256: summary.manifestSha256, artifacts: evidence.slice() };
+            }
+        }
+    }
+    plan.reused = retained.size;
+    plan.judgeCallsPlanned -= retained.size;
+    plan.ready = plan.issues.length === 0;
+    return { current, selected, plan, retained };
+}
+/** Preview eligible historical judgments. Reuse is not a fresh stability measurement. */
+export function planCalibration(loaded, options) {
+    return calibrationPlan(loaded, options).plan;
+}
+/** Explicit Pi-backed judging; semantic mismatches continue, infrastructure errors stop the remaining calls. */
+export async function runCalibration(loaded, options) {
+    const { current, selected, plan, retained } = calibrationPlan(loaded, options);
+    if (!plan.ready)
+        throw new Error(`Calibration is not ready: ${plan.issues.join("; ")}`);
     if (options.adapter.name !== "pi")
         throw new Error("Calibration requires Pi; direct provider adapters are unsupported");
     assertJudgeAllowed(options.judge, { source: "calibration judge", allowMetered: options.allowMeteredJudge });
-    if (!await options.adapter.available())
+    if (plan.judgeCallsPlanned && !await options.adapter.available())
         throw new Error("Pi is unavailable; no judge calls made");
     const output = resolve(options.output);
     mkdirSync(output, { mode: 0o700 }); // exclusive new directory, including when a previous run was interrupted
-    const save = (name, value) => writeFileSync(join(output, name), JSON.stringify(value, null, 2) + "\n", { flag: "wx", mode: 0o600 });
-    save("inputs.json", { manifestSha256: current.manifestSha256, judge: options.judge, controls: current.controls });
-    const summary = {
-        schema: 1, purpose: "judge-calibration", modelQualityScore: false, manifestSha256: current.manifestSha256,
-        judge: options.judge, cases: current.controls.length, completed: 0, matched: 0, judgeCalls: 0,
-        status: "matched", falsePasses: [], falseFailures: [], errors: [], unrun: [], output,
-    };
-    for (const [index, control] of current.controls.entries()) {
-        options.onProgress?.({ id: control.id, index, total: current.controls.length });
-        const scenario = current.spec.scenarios.find(s => s.id === control.scenarioId);
-        save(`${control.id}.request.json`, { prompt: control.prompt, sha256: hash(control.prompt) });
-        let attempt = 0;
-        const observedAdapter = {
-            name: "pi", available: () => options.adapter.available(), run: request => options.adapter.run(request),
-            judge: async (request) => {
-                const number = ++attempt;
-                save(`${control.id}.attempt-${number}.request.json`, { prompt: request.prompt, sha256: hash(request.prompt) });
-                summary.judgeCalls++;
-                const reply = await options.adapter.judge(request);
-                save(`${control.id}.attempt-${number}.reply.json`, { reply, sha256: hash(reply) });
-                return reply;
-            },
+    return withResourceMonitoring(output, async () => {
+        throwIfProcessAborted();
+        const save = (name, value) => writeFileSync(join(output, name), JSON.stringify(value, null, 2) + "\n", { flag: "wx", mode: 0o600 });
+        save("inputs.json", { manifestSha256: current.manifestSha256, judge: options.judge, controls: selected });
+        save("plan.json", plan);
+        const summary = {
+            schema: 1, purpose: "judge-calibration", modelQualityScore: false, manifestSha256: current.manifestSha256,
+            judge: options.judge, cases: selected.length, completed: 0, matched: 0, judgeCalls: 0,
+            calibrationScope: "answer-judge", executionGates: "not-evaluated", availableCases: current.controls.length,
+            partial: plan.partial, disagreements: [], reused: [],
+            status: "matched", falsePasses: [], falseFailures: [], errors: [], unrun: [], output,
         };
-        let result;
-        try {
-            result = await judgeInWorkspace(observedAdapter, options.judge, control.prompt, current.specDir, scenario.checklist.length);
+        for (const [index, control] of selected.entries()) {
+            throwIfProcessAborted();
+            const scenario = current.spec.scenarios.find(s => s.id === control.scenarioId);
+            let attempt = 0;
+            const observedAdapter = {
+                name: "pi", available: () => options.adapter.available(), run: request => options.adapter.run(request),
+                judge: async (request) => {
+                    throwIfProcessAborted();
+                    const number = ++attempt;
+                    save(`${control.id}.attempt-${number}.request.json`, { prompt: request.prompt, sha256: hash(request.prompt) });
+                    summary.judgeCalls++;
+                    const reply = await options.adapter.judge(request);
+                    save(`${control.id}.attempt-${number}.reply.json`, { reply, sha256: hash(reply) });
+                    throwIfProcessAborted();
+                    return reply;
+                },
+            };
+            let result;
+            try {
+                const prior = retained.get(control.id);
+                if (prior) {
+                    const source = plan.controls.find(row => row.id === control.id).source;
+                    result = prior.result;
+                    // Historical records remain byte-identical and self-contained, never counted as fresh calls.
+                    for (const [name, bytes] of prior.files)
+                        writeFileSync(join(output, name), bytes, { flag: "wx", mode: 0o600 });
+                    save(`${control.id}.reuse.json`, source);
+                    summary.reused.push({ id: control.id, source });
+                }
+                else {
+                    save(`${control.id}.request.json`, { prompt: control.prompt, sha256: hash(control.prompt) });
+                    options.onProgress?.({ id: control.id, index, total: selected.length });
+                    result = await judgeInWorkspace(observedAdapter, options.judge, control.prompt, current.specDir, scenario.checklist.length);
+                }
+            }
+            catch (error) {
+                if (isProcessControlError(error))
+                    throw error;
+                throwIfProcessAborted();
+                save(`${control.id}.error.json`, { status: "ERROR", message: String(error) });
+                summary.errors.push(control.id);
+                summary.unrun = selected.slice(index + 1).map(c => c.id);
+                break;
+            }
+            if (!retained.has(control.id))
+                save(`${control.id}.result.json`, { expected: control.expected, ...result });
+            if ((result.verdict !== "PASS" && result.verdict !== "FAIL") || result.suspect) {
+                summary.errors.push(control.id);
+                summary.unrun = selected.slice(index + 1).map(c => c.id);
+                break;
+            }
+            summary.completed++;
+            if (result.verdict === control.expected)
+                summary.matched++;
+            else {
+                summary.disagreements.push({ id: control.id, expected: control.expected, observed: result.verdict });
+                if (result.verdict === "PASS")
+                    summary.falsePasses.push(control.id);
+                else
+                    summary.falseFailures.push(control.id);
+            }
         }
-        catch (error) {
-            save(`${control.id}.error.json`, { status: "ERROR", message: String(error) });
-            summary.errors.push(control.id);
-            summary.unrun = current.controls.slice(index + 1).map(c => c.id);
-            break;
-        }
-        save(`${control.id}.result.json`, { expected: control.expected, ...result });
-        if ((result.verdict !== "PASS" && result.verdict !== "FAIL") || result.suspect) {
-            summary.errors.push(control.id);
-            summary.unrun = current.controls.slice(index + 1).map(c => c.id);
-            break;
-        }
-        summary.completed++;
-        if (result.verdict === control.expected)
-            summary.matched++;
-        else if (result.verdict === "PASS")
-            summary.falsePasses.push(control.id);
-        else
-            summary.falseFailures.push(control.id);
-    }
-    summary.status = summary.errors.length ? "error" : summary.matched === summary.cases ? "matched" : "mismatch";
-    save("summary.json", summary);
-    return summary;
+        summary.status = summary.errors.length ? "error" : summary.matched === summary.cases ? "matched" : "mismatch";
+        save("summary.json", summary);
+        return summary;
+    });
 }
 //# sourceMappingURL=calibration.js.map

package/dist/canary.d.ts

@@ -1,44 +1,46 @@
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-/**
- * The probe target: the longest `## ` heading in the skill's body.
- *
- * Body-only by construction — the frontmatter `description` is always in context
- * under progressive disclosure, so anything quotable from it would pass against a
- * model that never read the instructions. Longest rather than first because the
- * check is "did you see this text", and `## Overview` is guessable while
- * `## Refuse a metered judge, whatever chose it` is not.
- *
- * Null when the body has no `## ` heading: there is then nothing to ask for that a
- * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
- * worse than none.
- */
+/** A body-only quote target. Matching it is an observation, not proof of loading. */
 export declare function deliveryAnchor(skillMd: string): string | null;
+export interface CanaryObservations {
+    /** Local readability does not establish what Pi advertised to the subject. */
+    localSkillAvailable: boolean;
+    loadingRequested: boolean;
+    /** Successful completion of a read of the exact selected SKILL.md, if observable. */
+    successfulSkillRead: "observed" | "not-observed" | "unobserved";
+    /** Only the final assistant response is checked, never tool output or prompts. */
+    returnedAnchor: boolean | null;
+    /** The probe explicitly requests loading, so it cannot measure autonomous selection. */
+    autonomousSelection: "untested";
+    executionError?: string;
+}
 export interface CanaryResult {
-    /** `pass` = the anchor came back; `fail` = it did not; `skipped` = nothing safe to probe for. */
+    /** `fail` is an inconclusive delivery check, not evidence that the skill is unavailable. */
     status: "pass" | "fail" | "skipped";
-    /** What the probe looked for (null when skipped). */
     anchor: string | null;
-    /** Why it was skipped, or what the model said instead (trimmed) — for the log and the journal. */
     detail: string;
+    observations: CanaryObservations;
+    probeCalls: 0 | 1;
 }
-export declare function canaryPrompt(skillName: string, anchor: string): string;
+/**
+ * Ask Pi to load the named skill through its advertised discovery mechanism.
+ * Neither the expected quote nor a fallback file path is disclosed: a supplied
+ * path would let the probe bypass missing skill discovery entirely.
+ */
+export declare function canaryPrompt(skillName: string, _anchor?: string): string;
 export interface CanaryOptions {
     adapter: HarnessAdapter;
     model: ModelRef;
     skillDir: string;
     skillName: string;
-    /** Neutral cwd, same as a scenario gets — the probe must not see a repo either. */
+    /** Neutral cwd, same as a scenario gets. */
     cwd: string;
 }
 /**
- * Run the probe in green mode and report whether the skill body was reachable.
- *
- * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
- * with the reply in `detail`, because "the harness answered without the skill" and
- * "the model said something odd" are both reasons not to spend a wave. An adapter
- * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
- * setup errors with their own messages.
+ * One optional subject invocation, with read-only tools, before a green wave.
+ * A successful structured probe observes an instructed read and returned content;
+ * it does not establish autonomous skill selection in any later scenario. A
+ * failed quote is inconclusive, never proof that every later scenario is naked.
  */
 export declare function runDeliveryCanary(opts: CanaryOptions): Promise<CanaryResult>;
-/** The abort message for a failed canary: what was measured, and what to do instead. */
+/** Stop the requested wave without converting an inconclusive probe into a claim. */
 export declare function canaryFailure(skillName: string, result: CanaryResult, cliVersion: string | null): string;