@skill-harness/core 0.10.0 → 0.12.0

package/dist/qualification-validation.d.ts

@@ -0,0 +1,12 @@
+/** Canonical final validator: invocation evidence plus panel-board completeness. */
+export declare function validateQualificationRunnerSpool(spoolDir: string): {
+    derived: {
+        panels: number;
+        cells: number;
+    };
+    ok: true;
+    configuration_sha256: string;
+    invocations: number;
+    accounting: import("./qualification-store.js").QualificationAccountingReport;
+    terminal: number;
+};

package/dist/qualification-validation.js

@@ -0,0 +1,8 @@
+import { validateQualificationPanelOutputs } from "./qualification-panel-store.js";
+import { validateQualificationBaseSpool } from "./qualification-runner.js";
+/** Canonical final validator: invocation evidence plus panel-board completeness. */
+export function validateQualificationRunnerSpool(spoolDir) {
+    const base = validateQualificationBaseSpool(spoolDir);
+    return { ...base, derived: validateQualificationPanelOutputs(spoolDir) };
+}
+//# sourceMappingURL=qualification-validation.js.map

package/dist/regate.js

@@ -6,7 +6,7 @@ import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
 import { deserializeTrajectoryEvents, evaluateTrajectoryGates } from "./trajectory-gates.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, findTrajectoryFiles, tracePath, trajectoryPath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, } from "./results.js";
+import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, findTrajectoryFiles, tracePath, trajectoryPath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -222,9 +222,11 @@ export async function regateRun(opts) {
                     const priorStatus = objective?.status ?? "PASS";
                     const status = priorStatus === "ERROR" || g.status === "ERROR" || priorEvidenceErrors.length || digestMismatch
                         ? "ERROR"
-                        : priorStatus === "FAIL" || g.status === "FAIL"
-                            ? "FAIL"
-                            : "PASS";
+                        : priorStatus === "NOT-MEASURED"
+                            ? "NOT-MEASURED"
+                            : priorStatus === "FAIL" || g.status === "FAIL"
+                                ? "FAIL"
+                                : "PASS";
                     objective = {
                         ...(objective ?? {}),
                         status,
@@ -242,6 +244,21 @@ export async function regateRun(opts) {
                     }
                 }
             }
+            // Prompt delivery was observed during the subject invocation and cannot be
+            // replayed by regate. Carry its objective finding while refreshing only
+            // trace/trajectory/needle assertions.
+            const retainedDelivery = rec.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
+            const repObservations = prev.subject_invocations?.filter(observation => observation.scenario_id === scenario.id && observation.repetition === (rep ?? 0)) ?? [];
+            const delivery = prev.schema === 3
+                ? { kind: "skill_delivered", status: deliveryStatusForObservations(repObservations), detail: `${repObservations.length} retained provider request observation(s)` }
+                : retainedDelivery;
+            if (delivery) {
+                const prior = objective?.status ?? "PASS";
+                const status = delivery.status === "ERROR" || prior === "ERROR" ? "ERROR" : delivery.status === "NOT-MEASURED" ? "NOT-MEASURED" : prior === "FAIL" ? "FAIL" : "PASS";
+                objective = { ...(objective ?? {}), status, assertions: [delivery, ...(objective?.assertions ?? [])] };
+                if (delivery.status !== "PASS")
+                    traceFailure = `objective: ${delivery.detail}`;
+            }
             const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure ?? trajectoryFailure };
             const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
@@ -250,7 +267,7 @@ export async function regateRun(opts) {
             // trace gate leaves an `objective` block on the result. Reading only the
             // trailer meant a trace gate flipping to PASS never triggered the re-judge
             // it needs, leaving a stale FAIL verdict beside a PASS objective.
-            const oldObjectiveFailed = rec.objective?.status === "FAIL" || rec.objective?.status === "ERROR";
+            const oldObjectiveFailed = rec.objective?.status === "FAIL" || rec.objective?.status === "ERROR" || rec.objective?.status === "NOT-MEASURED";
             const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER))) || oldObjectiveFailed;
             // The trailer is regenerated whatever the outcome: leaving a stale
             // `MISSING` note beside a corrected verdict would misinform the next reader
@@ -258,11 +275,11 @@ export async function regateRun(opts) {
             if (existsSync(tPath))
                 rewriteTranscript(tPath, gate.lines);
             if (gate.failure) {
-                if (objective?.status === "ERROR")
+                if (objective?.status === "ERROR" || objective?.status === "NOT-MEASURED")
                     gateErroredHere = true;
                 else
                     gateFailedHere = true;
-                outcomes.push({ verdict: objective?.status === "ERROR" ? "ERROR" : "FAIL", reason: gate.failure, suspect: false, objective });
+                outcomes.push({ verdict: objective?.status === "ERROR" ? "ERROR" : objective?.status === "NOT-MEASURED" ? "NOT-MEASURED" : "FAIL", reason: gate.failure, suspect: false, objective });
                 continue;
             }
             if (!oldGateFailed) {
@@ -284,12 +301,26 @@ export async function regateRun(opts) {
         }
         const threshold = effectiveThreshold(rec, scenario);
         const next = outcomesToResult(scenario.id, outcomes, outcomes.length, threshold);
+        if (prev.schema === 3) {
+            next.criterion_count = scenario.checklist.length;
+            next.rep_judgments = outcomes.map((outcome, repetition) => {
+                const fresh = next.rep_judgments?.find(panel => panel.repetition === repetition);
+                const prior = rec.rep_judgments?.find(panel => panel.repetition === repetition);
+                const judgments = fresh?.judgments.length ? fresh.judgments : prior?.judgments ?? [];
+                return {
+                    repetition,
+                    judgments,
+                    recorded_verdict: outcome.verdict,
+                    ...(outcome.objective ? { objective: outcome.objective } : {}),
+                };
+            });
+        }
         next.metrics = mergeScenarioMetrics(rec.metrics, next.metrics);
         // Overrides and their notes survive: a regate re-decides the gate, and an author
         // override is a statement about the judge, not about the needle.
-        // `regate` re-evaluates gates from saved artifacts and asks no judge anything:
-        // `objective` is freshly recomputed above, and the recorded judge panel still
-        // describes the current judgments.
+        // `regate` re-evaluates gates from saved artifacts. It may judge a previously
+        // blocked fail→pass rep above; carrying the recorded panel still preserves any
+        // earlier multi-judge decision for judgments that were not replaced.
         scenarios.push(rebuildScenarioResult(next, rec, { objective: "fresh", adjudication: "carry" }));
         const to = next.judge_verdict;
         if (to !== rec.judge_verdict) {
@@ -302,9 +333,17 @@ export async function regateRun(opts) {
     }
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
+        schema: prev.schema, subject_invocations: prev.subject_invocations,
         skill: prev.skill, harness: prev.harness, model: prev.model,
         // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
         harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev.arm,
         judge: { provider: opts.judge.provider, model: opts.judge.model },
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode, partial: prev.partial,
         // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric

package/dist/regrade.d.ts

@@ -1,6 +1,6 @@
 import type { Spec, Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-import { type ScenarioResult, type ResultsFile } from "./results.js";
+import { type Judgment, type ScenarioResult, type ResultsFile } from "./results.js";
 import { type RepOutcome } from "./reps.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
@@ -84,4 +84,6 @@ export interface RegradeRunOptions {
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
+/** Preserve prior panel votes and append a full-cell grade without writing invalid one-vote history. */
+export declare function appendJudgeHistory(prior: ScenarioResult | undefined, priorJudge: ModelRef | undefined, fresh: Omit<Judgment, "ordinal">): Judgment[] | undefined;
 export declare function regradeRun(opts: RegradeRunOptions): Promise<ResultsFile>;

package/dist/regrade.js

@@ -1,10 +1,11 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, } from "./results.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
+import { providerFailureFromTranscript } from "./provider-failure.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -37,15 +38,32 @@ export function refreshRubricHashes(recorded, spec, judgedIds) {
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
     const startedAt = performance.now();
+    const repField = rep === undefined ? {} : { rep };
+    // A provider outage recorded in the transcript (either by `run()`'s marker or,
+    // since the fix for I2, by `runStructured` too) must stay ERROR through a
+    // re-judge. Without this check, `grade`/`regrade` reads the saved transcript
+    // with no infrastructure-failure guard at all — the one `run.ts` applies BEFORE
+    // ever calling this function — and re-judges "provider never ran the request"
+    // into a model verdict, spending a judge call to turn infrastructure noise into
+    // a false FAIL.
+    const providerFailure = providerFailureFromTranscript(transcript);
+    if (providerFailure) {
+        const reason = `provider failure — ${providerFailure}`;
+        appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
+        return {
+            verdict: "ERROR", reason, suspect: false,
+            metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 },
+        };
+    }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
-    const repField = rep === undefined ? {} : { rep };
     appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
     if (g.suspect)
         appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
     return {
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1,
@@ -95,6 +113,22 @@ export async function regradeScenario(opts) {
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
+/** Preserve prior panel votes and append a full-cell grade without writing invalid one-vote history. */
+export function appendJudgeHistory(prior, priorJudge, fresh) {
+    const history = prior?.judge_history ?? prior?.adjudication?.judgments ?? (prior && priorJudge ? [{
+            ordinal: 1,
+            judge: priorJudge,
+            verdict: prior.judge_verdict ?? "JUDGE-AMBIGUOUS",
+            reason: prior.judge_reason ?? "prior grade",
+            suspect: prior.suspect ?? true,
+            criteria: prior.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0]?.criteria ?? [],
+        }] : []);
+    const next = [
+        ...history,
+        { ...fresh, criteria: fresh.criteria ?? [], ordinal: history.length + 1 },
+    ].slice(-3).map((judgment, index) => ({ ...judgment, ordinal: index + 1 }));
+    return next.length >= 2 ? next : undefined;
+}
 export async function regradeRun(opts) {
     const { runDir, spec, adapter, judge, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
@@ -129,10 +163,16 @@ export async function regradeRun(opts) {
         targets = prev.scenarios
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
-        if (targets.length === 0) {
-            // Nothing untrustworthy — a no-op, not an error. Return the file as-is.
-            return prev;
-        }
+    }
+    // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
+    // cannot turn an undelivered or unauthenticated transcript into product evidence.
+    if (prev?.schema === 3) {
+        const blocked = new Set(prev.scenarios.filter(s => s.objective?.assertions.some(a => a.kind === "skill_delivered" && a.status !== "PASS")).map(s => s.id));
+        targets = targets.filter(id => !blocked.has(id));
+    }
+    if (targets.length === 0 && prev) {
+        // Nothing judgeable — a no-op, preserving both evidence and judge identity.
+        return prev;
     }
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
@@ -165,7 +205,14 @@ export async function regradeRun(opts) {
             expectedReps: prevScenario?.reps ?? 1,
         });
         const carry = overrides.get(id);
+        if (prev?.schema === 3)
+            rr.criterion_count = scenario.checklist.length;
         rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
+        rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
+        rr.judge_history = appendJudgeHistory(carry, prev?.judge, {
+            judge, verdict: rr.judge_verdict, reason: rr.judge_reason, suspect: rr.suspect,
+            criteria: rr.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0]?.criteria,
+        });
         // `grade` re-judges the saved transcript. It does not re-evaluate trace gates
         // (that is `regate`), so `objective` still describes this run; and it replaced
         // the judgments a prior adjudication described, so that panel must go.
@@ -173,6 +220,8 @@ export async function regradeRun(opts) {
     }
     const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
+        schema: prev?.schema,
+        subject_invocations: prev?.subject_invocations,
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
         // The harness CLI that produced these transcripts, carried verbatim: a re-grade
@@ -180,6 +229,13 @@ export async function regradeRun(opts) {
         // here would credit the old transcripts to a version that never ran them.
         harness_cli_version: prev?.harness_cli_version,
         delivery_canary: prev?.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev?.arm,
         model: prev?.model ?? "unknown",
         judge: { provider: judge.provider, model: judge.model },
         timestamp: prev?.timestamp ?? now(),

package/dist/reps.d.ts

@@ -1,5 +1,5 @@
 import type { Verdict } from "./score.js";
-import type { ScenarioResult, ObjectiveResult } from "./results.js";
+import type { ScenarioResult, ObjectiveResult, Judgment, SubjectInvocationObservation } from "./results.js";
 import type { TraceMetrics } from "./capture-trace-types.js";
 /** One rep's outcome (subject run + judge). */
 export interface RepOutcome {
@@ -8,6 +8,8 @@ export interface RepOutcome {
     suspect: boolean;
     /** Present only when the scenario declared structured objective assertions. */
     objective?: ObjectiveResult;
+    judgment?: Judgment;
+    subject_invocations?: SubjectInvocationObservation[];
     metrics?: {
         wall_time_ms: number;
         judge_calls: number;
@@ -22,7 +24,7 @@ export interface RepOutcome {
  * pass-threshold aggregation: an objective assertion is a statement about what
  * the model DID, so one rep that called a forbidden tool is a real finding, not
  * a minority draw to be voted away. ERROR dominates (missing evidence is never a
- * pass), then FAIL, then PASS. The retained assertion detail comes from the
+ * pass), then NOT-MEASURED (delivery absent), then FAIL, then PASS. The retained assertion detail comes from the
  * first non-passing rep, since that is the one worth reading.
  */
 export declare function aggregateObjective(outcomes: RepOutcome[]): ObjectiveResult | undefined;

package/dist/reps.js

@@ -5,14 +5,14 @@
  * pass-threshold aggregation: an objective assertion is a statement about what
  * the model DID, so one rep that called a forbidden tool is a real finding, not
  * a minority draw to be voted away. ERROR dominates (missing evidence is never a
- * pass), then FAIL, then PASS. The retained assertion detail comes from the
+ * pass), then NOT-MEASURED (delivery absent), then FAIL, then PASS. The retained assertion detail comes from the
  * first non-passing rep, since that is the one worth reading.
  */
 export function aggregateObjective(outcomes) {
     const present = outcomes.map((o) => o.objective).filter((o) => o !== undefined);
     if (present.length === 0)
         return undefined;
-    const picked = present.find((o) => o.status === "ERROR") ?? present.find((o) => o.status === "FAIL") ?? present[0];
+    const picked = present.find((o) => o.status === "ERROR") ?? present.find((o) => o.status === "NOT-MEASURED") ?? present.find((o) => o.status === "FAIL") ?? present[0];
     if (present.length === 1)
         return picked;
     const eventHashes = present.map((objective) => objective.events_sha256);
@@ -41,6 +41,10 @@ export function aggregateReps(outcomes, threshold) {
     if (errored > 0) {
         return { verdict: "ERROR", reason: `${errored}/${reps} reps errored — infrastructure, not behavior`, passes, reps, clean: clean.length, flakiness: 0, suspect: false };
     }
+    const notMeasured = outcomes.filter((o) => o.verdict === "NOT-MEASURED").length;
+    if (notMeasured > 0) {
+        return { verdict: "NOT-MEASURED", reason: `${notMeasured}/${reps} reps not measured — skill delivery was not established`, passes, reps, clean: clean.length - notMeasured, flakiness: 0, suspect: false };
+    }
     if (clean.length * 2 < reps) {
         // majority of reps misfired → untrustworthy
         return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
@@ -65,15 +69,17 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     const objectiveField = objective ? { objective } : {};
     const metrics = aggregateMetrics(outcomes);
     const metricsField = metrics ? { metrics } : {};
+    const repJudgments = outcomes.map((outcome, repetition) => ({ repetition, judgments: outcome.judgment ? [outcome.judgment] : [], recorded_verdict: outcome.verdict, ...(outcome.objective ? { objective: outcome.objective } : {}) }));
+    const repJudgmentField = outcomes.some(outcome => outcome.judgment) ? { rep_judgments: repJudgments } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, override: null, note: "", ...objectiveField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, override: null, note: "", ...objectiveField, ...repJudgmentField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, ...metricsField, override: null, note: "", ...objectiveField,
+        pass_threshold: threshold, ...metricsField, override: null, note: "", ...objectiveField, ...repJudgmentField,
     };
 }
 function aggregateMetrics(outcomes) {
@@ -97,6 +103,9 @@ function aggregateMetrics(outcomes) {
         cache_read_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_read_tokens, 0),
         cache_write_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_write_tokens, 0),
         subject_cost_usd: subjects.reduce((sum, metrics) => sum + metrics.cost_usd, 0),
+        cost_source: subjects.every((metrics) => metrics.cost_source === subjects[0].cost_source)
+            ? subjects[0].cost_source
+            : "unreported",
         tool_calls: subjects.reduce((sum, metrics) => sum + metrics.tool_calls, 0),
         delegated_children: subjects.reduce((sum, metrics) => sum + metrics.delegated_children, 0),
         max_concurrency: Math.max(...subjects.map((metrics) => metrics.max_concurrency)),

package/dist/rescore.js

@@ -52,7 +52,7 @@ export function rescoreRun(opts) {
         // no rate to re-apply, or an untrustworthy verdict → carry verbatim
         if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
             return s;
-        if (s.judge_verdict === "ERROR" || s.judge_verdict === "JUDGE-AMBIGUOUS")
+        if (s.judge_verdict === "ERROR" || s.judge_verdict === "NOT-MEASURED" || s.judge_verdict === "JUDGE-AMBIGUOUS")
             return s;
         if (s.clean === 0)
             return s;
@@ -78,6 +78,7 @@ export function rescoreRun(opts) {
     });
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
+        schema: prev.schema, subject_invocations: prev.subject_invocations,
         skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
         partial: prev.partial,
@@ -85,6 +86,13 @@ export function rescoreRun(opts) {
         // threshold to reps the recorded harness already produced.
         harness_cli_version: prev.harness_cli_version,
         delivery_canary: prev.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev.arm,
         // A rescore re-applies the CURRENT policy (thresholds, critical set) to the
         // recorded reps, so `policy:` drift is genuinely resolved by having run this —
         // that is what makes `rescore` the honest remedy lint names for it. Stimulus,

package/dist/results.d.ts

@@ -2,6 +2,7 @@ import { type ModelRef } from "./adapters/types.js";
 import { type ScenarioVerdict } from "./score.js";
 import type { Verdict } from "./score.js";
 import type { ShipBar, Scenario } from "./spec.js";
+import type { CostSource } from "./capture-trace-types.js";
 export interface ScenarioMetrics {
     wall_time_ms: number;
     judge_calls: number;
@@ -13,14 +14,36 @@ export interface ScenarioMetrics {
     cache_read_tokens?: number;
     cache_write_tokens?: number;
     subject_cost_usd?: number;
+    cost_source?: CostSource;
     tool_calls?: number;
     delegated_children?: number;
     max_concurrency?: number;
 }
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export declare function mergeScenarioMetrics(prior: ScenarioMetrics | undefined, fresh: ScenarioMetrics | undefined): ScenarioMetrics | undefined;
+export interface CriterionVote {
+    index: number;
+    verdict: "PASS" | "FAIL" | "ERROR";
+    reason: string;
+}
+export interface RepJudgmentPanel {
+    repetition: number;
+    judgments: Judgment[];
+    recorded_verdict: Verdict;
+    /** Per-repetition objective result; needed because scenario.objective is a strict aggregate. */
+    objective?: ObjectiveResult;
+}
+export declare function carryRepObjectives(fresh: RepJudgmentPanel[] | undefined, prior: RepJudgmentPanel[] | undefined): RepJudgmentPanel[] | undefined;
+export interface SubjectInvocationObservation {
+    scenario_id: string;
+    repetition: number;
+    attempt?: number;
+    prompt: import("./adapters/types.js").PromptProvenance;
+}
 export interface ScenarioResult {
     id: string;
+    /** Numbered rubric items expected on every schema-3 judgment. */
+    criterion_count?: number;
     judge_verdict: Verdict;
     judge_reason: string;
     suspect: boolean;
@@ -49,6 +72,10 @@ export interface ScenarioResult {
      * actually blocks SHIP; this field is the audit trail behind that flag.
      */
     adjudication?: AdjudicationResult;
+    /** Latest three full-cell grades; enables offline agreement reports with distinct judges. */
+    judge_history?: Judgment[];
+    /** Initial and panel judge members, by repetition, sufficient to recompute the recorded verdict. */
+    rep_judgments?: RepJudgmentPanel[];
 }
 /** One judge's answer for a cell, kept verbatim however the collapse turned out. */
 export interface Judgment {
@@ -62,8 +89,12 @@ export interface Judgment {
     reason: string;
     /** The judge misfired — recorded, never counted as a clean vote. */
     suspect: boolean;
+    /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
+    criteria?: CriterionVote[];
 }
 export interface AdjudicationResult {
+    /** The saved transcript repetition judged by every panel member (first rep by policy). */
+    repetition?: number;
     state: "confirmed" | "tie_broken" | "unresolved";
     /** Why the cell was re-judged. */
     trigger: "ambiguous" | "contradictory" | "non_unanimous" | "ship_deciding";
@@ -74,8 +105,8 @@ export interface AdjudicationResult {
 }
 /** Objective gate outcome for one scenario cell. */
 export interface ObjectiveResult {
-    /** ERROR means the evidence was missing or malformed — never a pass. */
-    status: "PASS" | "FAIL" | "ERROR";
+    /** ERROR is malformed evidence; NOT-MEASURED is a known-undelivered experiment. */
+    status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
     trace_version?: number;
     trace_sha256?: string;
     trajectory_version?: string;
@@ -85,7 +116,7 @@ export interface ObjectiveResult {
     rep_trace_sha256?: string[];
     assertions: {
         kind: string;
-        status: "PASS" | "FAIL" | "ERROR";
+        status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
         detail: string;
     }[];
 }
@@ -98,7 +129,7 @@ export interface GradeSummary {
     note: string;
 }
 export interface ResultsFile {
-    schema: 2;
+    schema: 2 | 3;
     /**
      * The harness version that wrote this file — provenance for the numbers in it.
      *
@@ -157,6 +188,39 @@ export interface ResultsFile {
     timestamp: string;
     label: string | null;
     mode: string;
+    /**
+     * The arm this run was measured under, absent for the control.
+     *
+     * Recorded rather than inferred from the tag: the tag says which arm, this says
+     * what the arm actually loaded — a run whose extension list or definition count
+     * differed from today's `arms.yaml` is not comparable, and only the record can
+     * show that.
+     *
+     * `ledger_events` is the arm's delivery proof, `--canary`'s equivalent for this
+     * axis: the pi-daddy ledger itself is gitignored (`*.jsonl`), so without a
+     * count here a vacuous arm run (extension loaded, nothing ever delegated)
+     * commits a record indistinguishable from one that actually exercised
+     * delegation. 0 is a reportable outcome, not an absent field.
+     *
+     * `env` is the arm's declared env for the subject process (`PI_GRANTS_GRANT`,
+     * `PI_GRANTS_MAX_DEPTH`, …) — recorded because it IS the experimental
+     * condition, not decoration around it. Without it, changing `PI_GRANTS_MAX_DEPTH`
+     * from 1 to 3 and re-running lands both runs in the same `+<arm>` tag with
+     * byte-identical `arm` records, and `stability` reads a verdict flip between two
+     * different conditions as one lineage being bimodal — the exact misreading the
+     * tag placement exists to prevent. Recorded as DECLARED, with `<run-dir>` and
+     * `<workspace>` unsubstituted: the substituted form differs every run for the same condition,
+     * which would make two identical conditions look like two.
+     *
+     * Optional only for records written before it was added; every writer emits it.
+     */
+    arm?: {
+        name: string;
+        extensions: string[];
+        definitions: number;
+        ledger_events: number;
+        env?: Record<string, string>;
+    };
     /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
     partial?: boolean;
     /**
@@ -171,6 +235,8 @@ export interface ResultsFile {
     source_hashes?: Record<string, string>;
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
+    /** Adapter-computed model-visible prompt observations. Required for schema v3. */
+    subject_invocations?: SubjectInvocationObservation[];
 }
 /**
  * The run modes whose results carry a real grade: the ones where the skill under
@@ -222,13 +288,28 @@ export declare function scoreContextFor(run: {
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export declare function effectiveThreshold(prevScenario: ScenarioResult | undefined, scenario: Scenario): number;
 /** Everything a caller may set. The grade is computed, never supplied. */
-export type ResultsDraft = Omit<ResultsFile, "schema" | "effective_grade">;
+export type ResultsDraft = Omit<ResultsFile, "schema" | "effective_grade"> & {
+    schema?: 2 | 3;
+};
 export interface ScoreContext {
     shipBar: ShipBar;
     critical: string[];
 }
-/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
-export declare function runDirFor(skillDir: string, harness: string, model: ModelRef, timestamp: string): string;
+/**
+ * `<skillDir>/tests/results/<harness>-<model-slug>[+<arm>]/<timestamp-slug>/`
+ *
+ * The arm is part of the run's IDENTITY, not of the spec's digest. `lint` and
+ * `stability` both key on this tag, so two arms become separate lineages that can
+ * never be misread as one lineage flipping run-over-run — and no
+ * `specification.yaml` byte moves, so every committed run stays valid.
+ *
+ * `+` is deliberate and is appended OUTSIDE `modelSlug`, whose character class
+ * (`[^A-Za-z0-9._-]`) cannot emit one. So the separator can never occur inside a
+ * slug and the tag stays unambiguously splittable back into model and arm. The
+ * control arm appends nothing, so its paths are byte-identical to before arms
+ * existed.
+ */
+export declare function runDirFor(skillDir: string, harness: string, model: ModelRef, timestamp: string, armName?: string): string;
 /** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
 export declare function transcriptPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 export declare function reportPath(runDir: string): string;
@@ -270,6 +351,17 @@ export declare function writeResults(runDir: string, draft: ResultsDraft, ctx: S
 export declare function migrateResults(raw: unknown): ResultsFile;
 /** Read results.yaml from a run dir, migrating schema-1 files in memory. */
 export declare function readResults(runDir: string): ResultsFile;
+/** Parse rubric-item votes for retention only; verdict semantics remain in grade.ts. */
+export declare function parseCriterionVotes(raw: string): CriterionVote[];
+export declare function completeCriterionVotes(votes: CriterionVote[], expected: number): CriterionVote[];
+export declare function deliveryStatusForObservations(observations: SubjectInvocationObservation[]): "PASS" | "NOT-MEASURED" | "ERROR";
+export declare function recomputeRecordedPanels(results: ResultsFile): Array<{
+    scenario_id: string;
+    repetition: number;
+    verdict: Verdict;
+}>;
+/** Runtime v2/v3 validator. v2 remains permissive; v3 fails closed on observation gaps. */
+export declare function validateResults(raw: unknown): ResultsFile;
 /** Pure: return a copy with override + note applied to one scenario. */
 export declare function applyOverride(results: ResultsFile, scenarioId: string, override: Verdict | null, note: string): ResultsFile;
 /**