@skill-harness/core 0.30.0 → 0.31.0

package/dist/canary.js

@@ -1,123 +1,140 @@
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-/**
- * The delivery canary: one cheap probe, before any scenario runs, that the skill
- * under test actually reached the model.
- *
- * Why this exists. `--mode green` asks the harness to activate the skill, and the
- * harness can decline without saying so. Measured on pi: 0.80.x wrapped the prompt
- * with the skill body; 0.83.0 switched to progressive disclosure (only the
- * description is in context, the body loads on demand — "models don't always do
- * this", pi's own docs); and a nonexistent `--skill` path is accepted silently,
- * exit 0 with a normal answer. The reference corpus ran two full waves in that
- * state: `architect` came back 7/14, ≈ its no-skill baseline, and looked entirely
- * plausible — the only tell was a contradictory failure mix (over-ceremony AND
- * capitulation at once) that no single skill edit produces.
- *
- * What it can and cannot prove. A pass means the skill's body is *reachable* in
- * this exact invocation — which is what kills the whole silent-non-delivery class:
- * a dropped flag, a wrong path, a harness that stopped honoring the mode. It does
- * NOT prove the body entered context for every later scenario; under progressive
- * disclosure that is the model's choice per turn, and no probe can promise it. The
- * mode whose delivery needs no promise is `force` (SKILL.md as system prompt),
- * which is why `run` recommends it rather than pretending the canary is equivalent.
- *
- * Cost: exactly one subject call per run, and only when asked for (`--canary`).
- * A run that aborts here has spent one rep instead of a wave.
- */
+import { readFileSync, realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
 /** Frontmatter-stripped body of a SKILL.md. */
 function skillBody(text) {
     const m = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(text);
     return m ? text.slice(m[0].length) : text;
 }
-/**
- * The probe target: the longest `## ` heading in the skill's body.
- *
- * Body-only by construction — the frontmatter `description` is always in context
- * under progressive disclosure, so anything quotable from it would pass against a
- * model that never read the instructions. Longest rather than first because the
- * check is "did you see this text", and `## Overview` is guessable while
- * `## Refuse a metered judge, whatever chose it` is not.
- *
- * Null when the body has no `## ` heading: there is then nothing to ask for that a
- * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
- * worse than none.
- */
+/** A body-only quote target. Matching it is an observation, not proof of loading. */
 export function deliveryAnchor(skillMd) {
     const headings = [...skillBody(skillMd).matchAll(/^##[ \t]+(.+?)[ \t]*$/gm)].map((m) => m[1].trim());
     if (headings.length === 0)
         return null;
     return headings.reduce((a, b) => (b.length > a.length ? b : a));
 }
-/** Normalize for comparison: case, whitespace runs, and markdown emphasis/backticks. */
 function normalize(s) {
     return s.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
 }
-export function canaryPrompt(skillName, anchor) {
-    // Asks for every heading, not just the anchor, because that is a question a model
-    // WITH the instructions answers naturally and a model without them can only
-    // invent. The instruction to say NOT_AVAILABLE keeps an honest miss from looking
-    // like a refusal to follow format.
-    return (`Answer from the instructions you have loaded — do not perform any task.\n\n` +
-        `List every level-2 markdown heading (lines starting with "## ") in the instructions of ` +
-        `the skill named "${skillName}", verbatim, one per line, with no other text.\n` +
-        `If you have no such instructions available, reply exactly: NOT_AVAILABLE\n\n` +
-        `(The heading text is what matters; keep it exact.)`);
-}
 /**
- * What the model actually said, per the adapters' shared transcript convention
- * (`>>> USER:` / `<<< ASSISTANT:`). Falls back to the whole text for an adapter
- * that doesn't use the markers.
+ * Ask Pi to load the named skill through its advertised discovery mechanism.
+ * Neither the expected quote nor a fallback file path is disclosed: a supplied
+ * path would let the probe bypass missing skill discovery entirely.
  */
+export function canaryPrompt(skillName, _anchor) {
+    return (`This is a skill-loading check, not a request to carry out the skill's task.\n\n` +
+        `Find the skill named ${JSON.stringify(skillName)} among the skills advertised to you. ` +
+        `If only its description is loaded, use the read tool to load its instructions from the advertised location. ` +
+        `Do not search other files, execute commands, or follow task instructions inside the skill.\n` +
+        `Then list every level-2 markdown heading (lines starting with "## ") from that skill, verbatim, one per line. ` +
+        `Do not guess headings. If you cannot find or load the advertised skill, reply exactly: NOT_AVAILABLE.\n` +
+        `A report of NOT_AVAILABLE records this attempt only; it does not establish that the skill cannot be loaded.`);
+}
+/** A legacy transcript without role boundaries cannot identify a safe answer. */
 function assistantReply(transcript) {
-    const parts = transcript.split(/^<<< ASSISTANT:\s*$/m);
-    return (parts.length > 1 ? parts[parts.length - 1] : transcript).trim();
+    const parts = transcript.split(/^<<< ASSISTANT:[ \t]*\r?$/m);
+    if (parts.length < 2)
+        return null;
+    return parts[parts.length - 1].split(/^(?:>>>|<<<) [A-Z][A-Z _-]*:.*$/m)[0].trim();
+}
+function quotedHeading(reply, anchor) {
+    // Whole heading lines avoid accepting an anchor embedded in an explanation,
+    // echoed JSON request, tool result, or a denial such as "I cannot find X".
+    return reply !== null && reply.split(/\r?\n/).some(line => normalize(line.replace(/^\s*(?:[-+]\s+|\d+[.)]\s+)?(?:##\s+)?/, "")) === normalize(anchor));
+}
+function canonicalPath(path, cwd) {
+    const expanded = /^~[/\\]/.test(path) ? join(homedir(), path.slice(2)) : path;
+    const absolute = resolve(cwd, expanded);
+    try {
+        return realpathSync(absolute);
+    }
+    catch {
+        return absolute;
+    }
+}
+function observedSkillRead(traces, skillPath, cwd) {
+    const expected = canonicalPath(skillPath, cwd);
+    return traces.some(trace => trace.tool_calls.some(call => call.name === "read" && call.isError === false && call.completionIndex >= 0 &&
+        call.result.bytes > 0 && typeof call.args.path === "string" &&
+        canonicalPath(call.args.path, cwd) === expected));
 }
 /**
- * Run the probe in green mode and report whether the skill body was reachable.
- *
- * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
- * with the reply in `detail`, because "the harness answered without the skill" and
- * "the model said something odd" are both reasons not to spend a wave. An adapter
- * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
- * setup errors with their own messages.
+ * One optional subject invocation, with read-only tools, before a green wave.
+ * A successful structured probe observes an instructed read and returned content;
+ * it does not establish autonomous skill selection in any later scenario. A
+ * failed quote is inconclusive, never proof that every later scenario is naked.
  */
 export async function runDeliveryCanary(opts) {
-    const skillMd = readFileSync(join(opts.skillDir, "SKILL.md"), "utf8");
+    const skillPath = join(opts.skillDir, "SKILL.md");
+    const observations = {
+        localSkillAvailable: false,
+        loadingRequested: false,
+        successfulSkillRead: "unobserved",
+        returnedAnchor: null,
+        autonomousSelection: "untested",
+    };
+    let skillMd;
+    try {
+        skillMd = readFileSync(skillPath, "utf8");
+        observations.localSkillAvailable = true;
+    }
+    catch {
+        return {
+            status: "fail", anchor: null, observations, probeCalls: 0,
+            detail: "The selected SKILL.md is not locally readable; no subject call was made.",
+        };
+    }
     const anchor = deliveryAnchor(skillMd);
-    if (!anchor) {
+    if (!anchor)
         return {
-            status: "skipped",
-            anchor: null,
-            detail: `${opts.skillName}/SKILL.md has no \`## \` heading to probe for — nothing a reply could prove`,
+            status: "skipped", anchor: null, observations, probeCalls: 0,
+            detail: `${opts.skillName}/SKILL.md has no \`## \` heading for this quote check; loading and selection remain untested`,
         };
+    observations.loadingRequested = true;
+    const request = {
+        skillDir: opts.skillDir, model: opts.model, mode: "green",
+        turns: [canaryPrompt(opts.skillName)], cwd: opts.cwd,
+        tools: ["read"], extensions: [], scenarioId: "(delivery-probe)", rep: 0,
+    };
+    let reply;
+    if (opts.adapter.runStructured) {
+        const run = await opts.adapter.runStructured(request);
+        const unhealthy = run.traces.some(trace => trace.capture_errors?.length ||
+            (trace.final_status !== undefined && trace.final_status !== "complete"));
+        if (run.providerFailure || run.executionFailure || unhealthy || run.traces.length !== 1) {
+            observations.executionError = run.providerFailure || run.executionFailure || "Incomplete or invalid structured probe evidence";
+            return { status: "fail", anchor, observations, probeCalls: 1,
+                detail: "The probe did not produce complete execution evidence; skill availability remains unresolved." };
+        }
+        reply = run.traces[0].final_text;
+        observations.successfulSkillRead = observedSkillRead(run.traces, skillPath, opts.cwd) ? "observed" : "not-observed";
+    }
+    else {
+        reply = assistantReply(await opts.adapter.run(request));
     }
-    const transcript = await opts.adapter.run({
-        skillDir: opts.skillDir,
-        model: opts.model,
-        mode: "green",
-        turns: [canaryPrompt(opts.skillName, anchor)],
-        cwd: opts.cwd,
-    });
-    const ok = normalize(transcript).includes(normalize(anchor));
+    observations.returnedAnchor = quotedHeading(reply, anchor);
+    const ok = observations.returnedAnchor && observations.successfulSkillRead !== "not-observed";
     return {
-        status: ok ? "pass" : "fail",
-        anchor,
-        // The reply, not the transcript: the transcript opens with our own prompt, and a
-        // failure report whose first 400 characters are the question is useless.
-        detail: ok ? "" : assistantReply(transcript).slice(0, 400),
+        status: ok ? "pass" : "fail", anchor, observations, probeCalls: 1,
+        detail: ok
+            ? (observations.successfulSkillRead === "observed"
+                ? "Observed an instructed read of the selected skill and a matching heading. Autonomous selection remains untested."
+                : "The final assistant response matched a heading; this adapter did not provide read evidence. Loading and autonomous selection remain unverified.")
+            : observations.returnedAnchor
+                ? "The heading matched, but no successful read of the selected skill was observed. A guess or preloaded context cannot establish instructed loading."
+                : `No matching heading was returned. This does not establish that the skill is unavailable. Reply: ${reply?.slice(0, 400) || "(unavailable)"}`,
     };
 }
-/** The abort message for a failed canary: what was measured, and what to do instead. */
+/** Stop the requested wave without converting an inconclusive probe into a claim. */
 export function canaryFailure(skillName, result, cliVersion) {
-    return (`delivery canary FAILED for ${skillName}: the model could not quote its own skill instructions ` +
-        `(looked for the heading \`${result.anchor}\`).\n` +
-        `  The skill is not reaching the model, so every scenario in this run would measure a naked ` +
-        `model and score like a result. Nothing has been spent beyond this one probe.\n` +
-        `  harness CLI: ${cliVersion ?? "unknown"}. On pi ≥ 0.83.0 \`--skill\` is progressive disclosure ` +
-        `(description in context, body on demand) and a nonexistent path is accepted silently.\n` +
-        `  Fix: re-run with \`--mode force\` (SKILL.md as the system prompt — delivery no version has made ` +
-        `conditional), or check that the skill dir is the one you meant.\n` +
-        `  What the model said instead: ${result.detail || "(nothing)"}`);
+    return (`delivery canary FAILED for ${skillName}: the requested loading check was inconclusive.\n` +
+        `  Local skill readable: ${result.observations.localSkillAvailable}; ` +
+        `successful skill read: ${result.observations.successfulSkillRead}; ` +
+        `matching heading: ${result.observations.returnedAnchor ?? "unobserved"}.\n` +
+        `  This does not prove skill unavailability or predict delivery in later scenarios. ` +
+        `Autonomous selection was not tested. Subject probe calls: ${result.probeCalls}.\n` +
+        `  harness CLI: ${cliVersion ?? "unknown"}. Check the skill path and recorded loading evidence, ` +
+        `or choose --mode force to evaluate explicit instruction delivery as a separate condition.\n` +
+        `  ${result.detail}`);
 }
 //# sourceMappingURL=canary.js.map

package/dist/diagnose.d.ts

@@ -1,9 +1,11 @@
+import { type SavedAnswerPlan } from "./saved-run.js";
 export interface Diagnosis {
     schema: 1;
     runDir: string;
     providerCalls: 0;
     resultsChanged: false;
     causalConclusion: "not-established";
+    reusePlan: SavedAnswerPlan;
     issues: {
         category: "evidence" | "configuration" | "judge" | "behavior" | "authoring";
         scenario?: string;

package/dist/diagnose.js

@@ -4,12 +4,13 @@ import { dirname, isAbsolute, relative, resolve } from "node:path";
 import { readResults, findTranscriptFiles } from "./results.js";
 import { loadSpec } from "./spec.js";
 import { currentHashFor, isSupersededKey, remedyForKey, scenarioIdForKey } from "./sources.js";
-import { inspectEvalAuthoring } from "./authoring-preflight.js";
+import { inspectEvalAuthoring, authoringEvidenceProfile, fixtureInventory } from "./authoring-preflight.js";
+import { planSavedAnswers } from "./saved-run.js";
 /** Explain the cheapest next investigation using saved records, never invent a root cause. */
 export function diagnoseRun(runDir, skillDir, specPath) {
     const results = readResults(runDir);
     const spec = loadSpec(specPath);
-    const report = { schema: 1, runDir: resolve(runDir), providerCalls: 0, resultsChanged: false, causalConclusion: "not-established", issues: [] };
+    const report = { schema: 1, runDir: resolve(runDir), providerCalls: 0, resultsChanged: false, causalConclusion: "not-established", reusePlan: planSavedAnswers(runDir, spec, skillDir, dirname(specPath)), issues: [] };
     const add = (issue) => report.issues.push(issue);
     const context = { skillDir, specDir: dirname(specPath), scenarios: spec.scenarios, judgePersona: spec.judge_persona };
     if (!results.source_hashes)
@@ -21,7 +22,15 @@ export function diagnoseRun(runDir, skillDir, specPath) {
         if (current === undefined || current !== previous)
             add({ category: "configuration", scenario: scenarioIdForKey(key, spec.scenarios), evidence: `${key}: ${current === undefined ? "not comparable" : "changed or unavailable"}`, next: remedyForKey(key) });
     }
-    for (const hint of inspectEvalAuthoring(spec))
+    for (const hint of inspectEvalAuthoring(spec, { ...authoringEvidenceProfile(dirname(specPath)), fixtureFiles: scenario => {
+            const fixture = typeof scenario.workspace === "object" ? scenario.workspace.fixture : scenario.fixture;
+            try {
+                return fixture ? fixtureInventory(resolve(dirname(specPath), fixture)) : [];
+            }
+            catch {
+                return [];
+            }
+        } }))
         add({ category: "authoring", scenario: hint.scenario, evidence: `${hint.code}: ${hint.message}`, next: "Review this contract using positive, negative and equivalent-answer controls; a heuristic is not proof the eval is wrong." });
     for (const scenario of results.scenarios) {
         const transcripts = findTranscriptFiles(runDir, scenario.id, results.mode);

package/dist/grade.js

@@ -3,6 +3,7 @@ import { completeCriterionVotes } from "./results.js";
 import { unavailableExecution } from "./judge-evidence.js";
 import { resolveJudgeReferences } from "./judge-references.js";
 export { resolveJudgeReferences } from "./judge-references.js";
+import { throwIfProcessAborted } from "./util/process.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -229,9 +230,11 @@ export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCrite
     const prompts = [];
     let validationError = "";
     for (let attempt = 0; attempt < 2; attempt++) {
+        throwIfProcessAborted();
         const retry = attempt === 0 ? "" : `\n\nYour reply was not valid: ${JSON.stringify(validationError)}; reply with only the JSON object.`;
         prompts.push(prompt + retry);
         const raw = await adapter.judge({ model: judge, prompt: prompts[prompts.length - 1], cwd });
+        throwIfProcessAborted();
         rawReplies.push(raw);
         // Pi adapter failures are infrastructure errors, not malformed model votes.
         if (/^\[judge error:/i.test(raw.trim())) {

package/dist/index.d.ts

@@ -24,6 +24,9 @@ export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/process.js";
+export * from "./resource-monitor.js";
+export * from "./saved-run.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";

package/dist/index.js

@@ -24,6 +24,9 @@ export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/process.js";
+export * from "./resource-monitor.js";
+export * from "./saved-run.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";

package/dist/journal.d.ts

@@ -1,4 +1,5 @@
 import type { Verdict } from "./score.js";
+import type { CanaryObservations } from "./canary.js";
 /**
  * Machine-facing event stream for one run: one JSON object per line in
  * <runDir>/journal.jsonl. UI, trends, and debugging read ONLY this (never
@@ -42,6 +43,8 @@ export type JournalEvent = {
     status: "pass" | "fail" | "skipped";
     anchor: string | null;
     detail: string;
+    observations?: CanaryObservations;
+    probeCalls?: 0 | 1;
 } | {
     event: "scenario-started";
     ts: string;

package/dist/lint.js

@@ -9,7 +9,7 @@ import { downgradeWarning } from "./downgrade.js";
 import { collectScoredRuns } from "./trends.js";
 import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
-import { fixtureInventory, inspectEvalAuthoring } from "./authoring-preflight.js";
+import { fixtureInventory, inspectEvalAuthoring, authoringEvidenceProfile } from "./authoring-preflight.js";
 import { resolveJudgeReferences } from "./judge-references.js";
 /** True when a finding fails the gate. The single place the exit-code rule lives. */
 export function failsGate(f) {
@@ -51,7 +51,7 @@ export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
         return [{ skill: basename(skillDir), code: "spec", message }];
     }
     const skill = spec.skill;
-    for (const hint of inspectEvalAuthoring(spec, { fixtureFiles: scenario => {
+    for (const hint of inspectEvalAuthoring(spec, { ...authoringEvidenceProfile(testsDir), fixtureFiles: scenario => {
             const fixture = effectiveFixture(scenario);
             if (!fixture)
                 return [];

package/dist/regate.js

@@ -364,6 +364,7 @@ export async function regateRun(opts) {
         skill: prev.skill, harness: prev.harness, model: prev.model,
         // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
         harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
+        delivery_observations: prev.delivery_observations,
         // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
         // only record that a `+<arm>` run actually delegated: rebuilding the draft
         // field-by-field without it silently deleted `definitions`/`ledger_events`

package/dist/regrade.d.ts

@@ -16,7 +16,7 @@ import { type JudgeExecutionFacts } from "./judge-evidence.js";
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[], specDir?: string, references?: Record<string, JudgeReference[]>): Record<string, string> | undefined;
+export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[], specDir?: string, references?: Record<string, JudgeReference[]>, refreshPersona?: boolean): Record<string, string> | undefined;
 export interface RegradeOptions {
     references?: JudgeReference[];
     runDir: string;
@@ -84,6 +84,8 @@ export interface RegradeRunOptions {
      * cleanly, so spending judge calls on them buys nothing.
      */
     onlySuspect?: boolean;
+    /** Rejudge only these recorded scenarios; untouched verdicts remain historical. */
+    only?: string[];
     /** Rejudge only retained repetitions containing ERROR criterion votes, with the recorded judge. */
     onlyUnparsed?: boolean;
 }

package/dist/regrade.js

@@ -9,6 +9,8 @@ import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
 import { loadExecutionFacts, transcriptResponses } from "./judge-evidence.js";
+import { withResourceMonitoring } from "./resource-monitor.js";
+import { throwIfProcessAborted } from "./util/process.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -21,7 +23,7 @@ import { loadExecutionFacts, transcriptResponses } from "./judge-evidence.js";
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export function refreshRubricHashes(recorded, spec, judgedIds, specDir, references) {
+export function refreshRubricHashes(recorded, spec, judgedIds, specDir, references, refreshPersona = true) {
     if (!recorded)
         return undefined;
     const next = { ...recorded };
@@ -33,7 +35,7 @@ export function refreshRubricHashes(recorded, spec, judgedIds, specDir, referenc
         if (s && RUBRIC_PREFIX + id in next)
             next[RUBRIC_PREFIX + id] = rubricDigest(s, specDir, references?.[id]);
     }
-    if (PERSONA_KEY in next)
+    if (refreshPersona && PERSONA_KEY in next)
         next[PERSONA_KEY] = personaDigest(spec.judge_persona);
     return next;
 }
@@ -177,6 +179,7 @@ function hasUnparsedVotes(panel) {
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
 export async function regradeRun(opts) {
+    throwIfProcessAborted();
     const { runDir, spec, adapter, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
@@ -208,15 +211,23 @@ export async function regradeRun(opts) {
     const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
     const recorded = prev?.scenarios ?? spec.scenarios.map((s) => ({ id: s.id }));
     let targets = recorded.map((s) => s.id);
+    if (opts.only !== undefined) {
+        if (!opts.only.length || opts.only.some(id => !id.trim()) || new Set(opts.only).size !== opts.only.length)
+            throw new Error("Selected scenarios must be a nonempty unique ID list");
+        const unknown = opts.only.filter(id => !targets.includes(id));
+        if (unknown.length)
+            throw new Error(`Unknown recorded scenario(s): ${unknown.join(", ")}`);
+        targets = targets.filter(id => opts.only.includes(id));
+    }
     if (opts.onlySuspect) {
         if (!prev)
             throw new Error(`--suspect-only needs a prior results.yaml in ${runDir}`);
-        targets = prev.scenarios
+        targets = prev.scenarios.filter(s => targets.includes(s.id))
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
     }
     if (opts.onlyUnparsed)
-        targets = recorded.filter(s => s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
+        targets = recorded.filter(s => targets.includes(s.id) && s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
     // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
     // cannot turn an undelivered or unauthenticated transcript into product evidence.
     if (prev?.schema === 3) {
@@ -227,6 +238,8 @@ export async function regradeRun(opts) {
         // Nothing judgeable — a no-op, preserving both evidence and judge identity.
         return prev;
     }
+    if (prev && targets.length !== recorded.length && (judge.provider !== prev.judge.provider || judge.model !== prev.judge.model))
+        throw new Error("Selective regrade must use the recorded judge; regrade the whole run to change judge");
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
         const files = findTranscriptFiles(runDir, record.id, mode);
@@ -266,75 +279,79 @@ export async function regradeRun(opts) {
         }
     }
     const references = Object.fromEntries(targets.map(id => [id, resolveJudgeReferences(specById.get(id), specDir)]));
-    const targetSet = new Set(targets);
-    const scenarioResults = [];
-    for (const rec of recorded) {
-        const id = rec.id;
-        if (!targetSet.has(id)) {
-            scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
-            continue;
+    return withResourceMonitoring(runDir, async () => {
+        const targetSet = new Set(targets);
+        const scenarioResults = [];
+        for (const rec of recorded) {
+            throwIfProcessAborted();
+            const id = rec.id;
+            if (!targetSet.has(id)) {
+                scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
+                continue;
+            }
+            const scenario = specById.get(id); // guaranteed present by the guard above
+            const prevScenario = prev?.scenarios.find((s) => s.id === id);
+            const threshold = effectiveThreshold(prevScenario, scenario);
+            const rr = await regradeScenario({
+                runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
+                references: references[id],
+                expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
+                ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version,
+            });
+            const carry = overrides.get(id);
+            if (prev?.schema === 3)
+                rr.criterion_count = scenario.checklist.length;
+            rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
+            rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
+            // Gates still describe the original subject execution. An adjudication of
+            // an untouched repetition also survives, bounded by rebuild's conflict guards.
+            const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
+            scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
         }
-        const scenario = specById.get(id); // guaranteed present by the guard above
-        const prevScenario = prev?.scenarios.find((s) => s.id === id);
-        const threshold = effectiveThreshold(prevScenario, scenario);
-        const rr = await regradeScenario({
-            runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
-            references: references[id],
-            expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
-            ...(prev?.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev?.harness_cli_version,
-        });
-        const carry = overrides.get(id);
-        if (prev?.schema === 3)
-            rr.criterion_count = scenario.checklist.length;
-        rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
-        rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
-        // Gates still describe the original subject execution. An adjudication of
-        // an untouched repetition also survives, bounded by rebuild's conflict guards.
-        const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
-        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
-    }
-    const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
-    const results = writeResults(runDir, {
-        schema: prev?.schema,
-        subject_invocations: prev?.subject_invocations,
-        skill: spec.skill,
-        harness: prev?.harness ?? "pi",
-        // The harness CLI that produced these transcripts, carried verbatim: a re-grade
-        // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
-        // here would credit the old transcripts to a version that never ran them.
-        harness_cli_version: prev?.harness_cli_version,
-        delivery_canary: prev?.delivery_canary,
-        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
-        // only record that a `+<arm>` run actually delegated: rebuilding the draft
-        // field-by-field without it silently deleted `definitions`/`ledger_events`
-        // from any arm run that was ever re-graded, leaving a record
-        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
-        // `delivery_canary` and `source_hashes` are carried here.
-        arm: prev?.arm,
-        model: prev?.model ?? "unknown",
-        judge: { provider: judge.provider, model: judge.model },
-        timestamp: prev?.timestamp ?? now(),
-        label: prev?.label ?? null,
-        mode,
-        // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
-        // the recorded **stimulus** hashes stay, keeping an honestly-stale run honestly
-        // stale. The rubric hashes are a different matter: this re-grade applied the
-        // CURRENT checklist and persona to those transcripts, so "the verdicts reflect
-        // today's rubric" is now a true statement about the record, and the hashes should
-        // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
-        // *stimulus* hashes stay" — see refreshRubricHashes.
-        partial: prev?.partial,
-        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets, opts.specDir, references),
-        source_hash_roots: prev?.source_hash_roots,
-        scenarios: scenarioResults,
-    }, ctx);
-    const g = results.effective_grade;
-    if (ctx) {
-        appendJournal(runDir, {
-            event: "score", ts: now(),
-            passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note,
-        });
-    }
-    return results;
+        const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
+        const results = writeResults(runDir, {
+            schema: prev?.schema,
+            subject_invocations: prev?.subject_invocations,
+            skill: spec.skill,
+            harness: prev?.harness ?? "pi",
+            // The harness CLI that produced these transcripts, carried verbatim: a re-grade
+            // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
+            // here would credit the old transcripts to a version that never ran them.
+            harness_cli_version: prev?.harness_cli_version,
+            delivery_canary: prev?.delivery_canary,
+            delivery_observations: prev?.delivery_observations,
+            // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+            // only record that a `+<arm>` run actually delegated: rebuilding the draft
+            // field-by-field without it silently deleted `definitions`/`ledger_events`
+            // from any arm run that was ever re-graded, leaving a record
+            // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+            // `delivery_canary` and `source_hashes` are carried here.
+            arm: prev?.arm,
+            model: prev?.model ?? "unknown",
+            judge: { provider: judge.provider, model: judge.model },
+            timestamp: prev?.timestamp ?? now(),
+            label: prev?.label ?? null,
+            mode,
+            // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
+            // the recorded **stimulus** hashes stay, keeping an honestly-stale run honestly
+            // stale. The rubric hashes are a different matter: this re-grade applied the
+            // CURRENT checklist and persona to those transcripts, so "the verdicts reflect
+            // today's rubric" is now a true statement about the record, and the hashes should
+            // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
+            // *stimulus* hashes stay" — see refreshRubricHashes.
+            partial: prev?.partial,
+            source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets, opts.specDir, references, targets.length === recorded.length),
+            source_hash_roots: prev?.source_hash_roots,
+            scenarios: scenarioResults,
+        }, ctx);
+        const g = results.effective_grade;
+        if (ctx) {
+            appendJournal(runDir, {
+                event: "score", ts: now(),
+                passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note,
+            });
+        }
+        return results;
+    });
 }
 //# sourceMappingURL=regrade.js.map

package/dist/rescore.js

@@ -95,6 +95,7 @@ export function rescoreRun(opts) {
         // threshold to reps the recorded harness already produced.
         harness_cli_version: prev.harness_cli_version,
         delivery_canary: prev.delivery_canary,
+        delivery_observations: prev.delivery_observations,
         // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
         // only record that a `+<arm>` run actually delegated: rebuilding the draft
         // field-by-field without it silently deleted `definitions`/`ledger_events`

package/dist/resource-monitor.d.ts

@@ -0,0 +1,3 @@
+import { type ProcessScope } from "./util/process.js";
+/** Per-operation metadata only, with bounded retention and low-frequency atomic snapshots. */
+export declare function withResourceMonitoring<T>(directory: string, work: () => Promise<T>, options?: ProcessScope): Promise<T>;