@skill-harness/core 0.11.0 → 0.12.0

package/dist/results.js

@@ -4,6 +4,7 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+import { collapseVotePanel } from "./vote-panel.js";
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export function mergeScenarioMetrics(prior, fresh) {
     if (!prior)
@@ -22,11 +23,20 @@ export function mergeScenarioMetrics(prior, fresh) {
         ...(subject.cache_read_tokens === undefined ? {} : { cache_read_tokens: subject.cache_read_tokens }),
         ...(subject.cache_write_tokens === undefined ? {} : { cache_write_tokens: subject.cache_write_tokens }),
         ...(subject.subject_cost_usd === undefined ? {} : { subject_cost_usd: subject.subject_cost_usd }),
+        ...(subject.cost_source === undefined ? {} : { cost_source: subject.cost_source }),
         ...(subject.tool_calls === undefined ? {} : { tool_calls: subject.tool_calls }),
         ...(subject.delegated_children === undefined ? {} : { delegated_children: subject.delegated_children }),
         ...(subject.max_concurrency === undefined ? {} : { max_concurrency: subject.max_concurrency }),
     };
 }
+export function carryRepObjectives(fresh, prior) {
+    if (!fresh)
+        return prior;
+    return fresh.map(panel => {
+        const objective = prior?.find(candidate => candidate.repetition === panel.repetition)?.objective;
+        return objective ? { ...panel, objective } : panel;
+    });
+}
 /**
  * The run modes whose results carry a real grade: the ones where the skill under
  * test was actually delivered to the model.
@@ -159,6 +169,8 @@ function objectiveVerdict(s) {
         return undefined;
     if (s.objective.status === "ERROR")
         return "ERROR";
+    if (s.objective.status === "NOT-MEASURED")
+        return "NOT-MEASURED";
     if (s.objective.status === "FAIL")
         return "FAIL";
     return undefined;
@@ -179,8 +191,9 @@ export function finalizeResults(draft, ctx) {
         const why = draft.partial ? "partial run (--only) — not scored" : `mode=${draft.mode} (not scored)`;
         effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: why };
     }
+    const schema = draft.schema ?? (draft.subject_invocations ? 3 : 2);
     return {
-        schema: 2,
+        schema,
         // Stamped here, the single place every writer passes through, so `run`,
         // `grade`, `rescore` and the review UI's override save all record which tool
         // produced the record they leave behind.
@@ -201,11 +214,13 @@ export function finalizeResults(draft, ctx) {
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         effective_grade,
         scenarios: draft.scenarios,
+        ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),
     };
 }
 /** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
 export function writeResults(runDir, draft, ctx) {
     const results = finalizeResults(draft, ctx);
+    validateResults(results);
     mkdirSync(runDir, { recursive: true });
     writeFileSync(resultsPath(runDir), yaml.dump(results, { lineWidth: 100 }), "utf8");
     return results;
@@ -217,8 +232,8 @@ export function migrateResults(raw) {
         throw new Error("empty or invalid results.yaml");
     }
     const o = raw;
-    if (o.schema === 2)
-        return raw;
+    if (o.schema === 2 || o.schema === 3)
+        return validateResults(raw);
     const v1 = raw;
     const modeMatch = /^mode=(\w+)/.exec(v1.grade?.note ?? "");
     return {
@@ -250,6 +265,191 @@ export function readResults(runDir) {
     const text = readFileSync(resultsPath(runDir), "utf8");
     return migrateResults(yaml.load(text));
 }
+const CRITERION_RE = /^\s*(\d+)[.)]\s*\**\s*(PASS|FAIL)\b\**\s*(.*)$/gim;
+/** Parse rubric-item votes for retention only; verdict semantics remain in grade.ts. */
+export function parseCriterionVotes(raw) {
+    return [...raw.matchAll(CRITERION_RE)].map((match) => ({
+        index: Number(match[1]), verdict: match[2].toUpperCase(),
+        reason: match[3].trim().replace(/^[-—:]\s*/, ""),
+    }));
+}
+export function completeCriterionVotes(votes, expected) {
+    return Array.from({ length: expected }, (_, offset) => votes.find(vote => vote.index === offset + 1) ?? {
+        index: offset + 1, verdict: "ERROR", reason: "judge emitted no parseable vote for this criterion",
+    });
+}
+export function deliveryStatusForObservations(observations) {
+    if (observations.length === 0)
+        return "ERROR";
+    const terminalAttempt = Math.max(...observations.map(observation => observation.attempt ?? 0));
+    const terminal = observations.filter(observation => (observation.attempt ?? 0) === terminalAttempt);
+    if (terminal.some(observation => observation.prompt.status === "ERROR"))
+        return "ERROR";
+    const mechanism = terminal[0].prompt.mechanism;
+    if (terminal.some(observation => observation.prompt.mechanism !== mechanism))
+        return "ERROR";
+    const counts = terminal.map(observation => observation.prompt.contract_occurrences);
+    if (mechanism === "none")
+        return counts.every(count => count === 0) ? "PASS" : "NOT-MEASURED";
+    if (mechanism === "pi-skill") {
+        const firstDelivered = counts.indexOf(1);
+        return firstDelivered >= 0 && counts.slice(0, firstDelivered).every(count => count === 0) && counts.slice(firstDelivered).every(count => count === 1) ? "PASS" : "NOT-MEASURED";
+    }
+    return counts.every(count => count === 1) ? "PASS" : "NOT-MEASURED";
+}
+export function recomputeRecordedPanels(results) {
+    const out = [];
+    for (const scenario of results.scenarios)
+        for (const panel of scenario.rep_judgments ?? []) {
+            const clean = panel.judgments.filter(j => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
+            let verdict = panel.recorded_verdict;
+            if (clean.length === 1)
+                verdict = clean[0].verdict;
+            else if (clean.length >= 2)
+                verdict = collapseVotePanel(panel.judgments).verdict ?? "JUDGE-AMBIGUOUS";
+            out.push({ scenario_id: scenario.id, repetition: panel.repetition, verdict });
+        }
+    return out;
+}
+/** Runtime v2/v3 validator. v2 remains permissive; v3 fails closed on observation gaps. */
+export function validateResults(raw) {
+    if (!raw || typeof raw !== "object")
+        throw new Error("invalid results");
+    const result = raw;
+    if (result.schema !== 2 && result.schema !== 3)
+        throw new Error(`unsupported results schema ${String(raw.schema)}`);
+    if (!Array.isArray(result.scenarios))
+        throw new Error("results scenarios missing");
+    if (result.schema === 2)
+        return result;
+    if (!Array.isArray(result.subject_invocations))
+        throw new Error("schema v3 delivery observations missing");
+    for (const observation of result.subject_invocations) {
+        const p = observation?.prompt;
+        if (!p || !["PASS", "NOT-MEASURED", "ERROR"].includes(p.status))
+            throw new Error("delivery status missing");
+        if (p.normalization_rule !== "cwd-line-v1")
+            throw new Error("unknown prompt normalization rule");
+        if (!/^[a-f0-9]{64}$/i.test(p.raw_sha256) || !/^[a-f0-9]{64}$/i.test(p.normalized_sha256) || !/^[a-f0-9]{64}$/i.test(p.contract_sha256))
+            throw new Error("invalid prompt provenance digest");
+        if (!Number.isInteger(p.bytes) || p.bytes < 0 || !Number.isInteger(p.contract_bytes) || p.contract_bytes < 0)
+            throw new Error("invalid prompt provenance byte length");
+        if (!Number.isInteger(p.contract_occurrences) || p.contract_occurrences < 0)
+            throw new Error("invalid delivery occurrence count");
+        const expected = p.mechanism === "none" ? 0 : 1;
+        const computed = p.contract_occurrences === expected ? "PASS" : "NOT-MEASURED";
+        if (p.status !== "ERROR" && p.status !== computed)
+            throw new Error("delivery status contradicts occurrence count");
+    }
+    const scenarioIds = new Set(result.scenarios.map(scenario => scenario.id));
+    const repsByScenario = new Map(result.scenarios.map(scenario => [scenario.id, scenario.reps ?? 1]));
+    if (result.subject_invocations.some(observation => !scenarioIds.has(observation.scenario_id) || !Number.isInteger(observation.repetition) || observation.repetition < 0 || observation.repetition >= (repsByScenario.get(observation.scenario_id) ?? 0)))
+        throw new Error("schema v3 contains orphan or out-of-range subject invocation observation");
+    const recomputed = recomputeRecordedPanels(result);
+    for (const scenario of result.scenarios) {
+        const reps = scenario.reps ?? 1;
+        if (!Array.isArray(scenario.rep_judgments) || scenario.rep_judgments.length !== reps)
+            throw new Error(`schema v3 repetition judgments missing for ${scenario.id}`);
+        const panelRepetitions = scenario.rep_judgments.map(panel => panel.repetition).sort((a, b) => a - b);
+        if (panelRepetitions.some((repetition, index) => repetition !== index))
+            throw new Error(`schema v3 repetition judgments duplicate, missing, or out of range for ${scenario.id}`);
+        for (let repetition = 0; repetition < reps; repetition++) {
+            const observations = result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === repetition);
+            // A pre-provider infrastructure failure legitimately has zero requests;
+            // its per-repetition skill_delivered objective must remain explicit ERROR.
+            const attempts = [...new Set(observations.map(observation => observation.attempt ?? 0))].sort((a, b) => a - b);
+            if (attempts.some((attempt, index) => !Number.isInteger(attempt) || attempt !== index))
+                throw new Error(`delivery attempts are duplicate, missing, or out of range for ${scenario.id}#${repetition}`);
+            for (const attempt of attempts) {
+                const indexes = observations.filter(observation => (observation.attempt ?? 0) === attempt).map(observation => observation.prompt.request_index).sort((a, b) => a - b);
+                if (indexes.some((requestIndex, index) => !Number.isInteger(requestIndex) || requestIndex !== index))
+                    throw new Error(`prompt request indexes are duplicate, missing, or out of range for ${scenario.id}#${repetition} attempt ${attempt}`);
+            }
+        }
+        const repStatuses = Array.from({ length: reps }, (_, repetition) => deliveryStatusForObservations(result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === repetition)));
+        const expectedStatus = repStatuses.includes("ERROR") ? "ERROR" : repStatuses.includes("NOT-MEASURED") ? "NOT-MEASURED" : "PASS";
+        const gate = scenario.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
+        if (!gate || gate.status !== expectedStatus)
+            throw new Error(`skill_delivered gate missing or inconsistent for ${scenario.id}`);
+        if (!Number.isInteger(scenario.criterion_count) || scenario.criterion_count < 1)
+            throw new Error(`schema v3 criterion count missing for ${scenario.id}`);
+        const assertCriteria = (judgment) => {
+            if (!Array.isArray(judgment.criteria))
+                throw new Error("schema v3 criterion votes missing");
+            const indexes = judgment.criteria.map(vote => vote.index).sort((a, b) => a - b);
+            if (indexes.length !== scenario.criterion_count || indexes.some((index, offset) => index !== offset + 1))
+                throw new Error(`schema v3 criterion votes are not complete contiguous indexes for ${scenario.id}`);
+        };
+        for (const panel of scenario.rep_judgments) {
+            for (const judgment of panel.judgments)
+                assertCriteria(judgment);
+            const cleanJudgments = panel.judgments.filter(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL"));
+            const objectiveBehavioralFail = panel.objective?.status === "FAIL" && panel.recorded_verdict === "FAIL";
+            if ((panel.recorded_verdict === "PASS" || panel.recorded_verdict === "FAIL") && cleanJudgments.length === 0 && !objectiveBehavioralFail)
+                throw new Error(`recorded behavioral verdict is unsupported by a clean judgment for ${scenario.id}#${panel.repetition}`);
+            const actual = recomputed.find(x => x.scenario_id === scenario.id && x.repetition === panel.repetition);
+            if (panel.judgments.length > 0 && actual.verdict !== panel.recorded_verdict)
+                throw new Error(`recorded panel verdict diverges from recomputed votes for ${scenario.id}#${panel.repetition}`);
+            if (!panel.objective)
+                throw new Error(`schema v3 per-repetition objective missing for ${scenario.id}#${panel.repetition}`);
+            const observations = result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === panel.repetition);
+            const deliveryStatus = deliveryStatusForObservations(observations);
+            const delivery = panel.objective.assertions.find(assertion => assertion.kind === "skill_delivered");
+            if (!delivery || delivery.status !== deliveryStatus)
+                throw new Error(`per-repetition skill_delivered gate missing or inconsistent for ${scenario.id}#${panel.repetition}`);
+            if (deliveryStatus === "ERROR" && panel.objective.status !== "ERROR")
+                throw new Error(`per-repetition objective weakens delivery ERROR for ${scenario.id}#${panel.repetition}`);
+            if (deliveryStatus === "NOT-MEASURED" && !["ERROR", "NOT-MEASURED"].includes(panel.objective.status))
+                throw new Error(`per-repetition objective weakens delivery NOT-MEASURED for ${scenario.id}#${panel.repetition}`);
+            const objectiveVerdict = panel.objective.status === "ERROR" ? "ERROR" : panel.objective.status === "NOT-MEASURED" ? "NOT-MEASURED" : panel.objective.status === "FAIL" ? "FAIL" : undefined;
+            if (objectiveVerdict && panel.recorded_verdict !== objectiveVerdict)
+                throw new Error(`recorded panel verdict weakens objective evidence for ${scenario.id}#${panel.repetition}`);
+        }
+        const objectiveStatuses = scenario.rep_judgments.map(panel => panel.objective.status);
+        const aggregateObjectiveStatus = objectiveStatuses.includes("ERROR") ? "ERROR" : objectiveStatuses.includes("NOT-MEASURED") ? "NOT-MEASURED" : objectiveStatuses.includes("FAIL") ? "FAIL" : "PASS";
+        if (!scenario.objective || scenario.objective.status !== aggregateObjectiveStatus)
+            throw new Error(`scenario objective diverges from per-repetition objectives for ${scenario.id}`);
+        for (const judgment of [...(scenario.judge_history ?? []), ...(scenario.adjudication?.judgments ?? [])])
+            assertCriteria(judgment);
+        let adjudicatedVerdict;
+        if (scenario.adjudication) {
+            if (!Number.isInteger(scenario.adjudication.repetition) || scenario.adjudication.repetition < 0 || scenario.adjudication.repetition >= reps)
+                throw new Error(`schema v3 adjudication repetition missing or out of range for ${scenario.id}`);
+            const collapsed = collapseVotePanel(scenario.adjudication.judgments);
+            const boundedCriticalAggregate = (scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
+            if (!boundedCriticalAggregate && (scenario.adjudication.state !== collapsed.state || scenario.adjudication.verdict !== collapsed.verdict))
+                throw new Error(`recorded adjudication state/verdict diverges from recomputed votes for ${scenario.id}`);
+            if (scenario.adjudication.state !== "unresolved")
+                adjudicatedVerdict = scenario.adjudication.verdict;
+        }
+        const panels = scenario.rep_judgments;
+        const errorCount = panels.filter(panel => panel.recorded_verdict === "ERROR").length;
+        const notMeasuredCount = panels.filter(panel => panel.recorded_verdict === "NOT-MEASURED").length;
+        const cleanPanels = panels.filter(panel => !(panel.judgments[0]?.suspect ?? false));
+        const passes = cleanPanels.filter(panel => panel.recorded_verdict === "PASS").length;
+        let aggregateVerdict;
+        let aggregateSuspect = false;
+        if (panels.length === 1) {
+            aggregateVerdict = panels[0].recorded_verdict;
+            aggregateSuspect = panels[0].judgments[0]?.suspect ?? false;
+        }
+        else if (errorCount > 0)
+            aggregateVerdict = "ERROR";
+        else if (notMeasuredCount > 0)
+            aggregateVerdict = "NOT-MEASURED";
+        else if (cleanPanels.length * 2 < panels.length) {
+            aggregateVerdict = "FAIL";
+            aggregateSuspect = true;
+        }
+        else
+            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) ? "PASS" : "FAIL";
+        const expectedVerdict = adjudicatedVerdict ?? aggregateVerdict;
+        const expectedSuspect = scenario.adjudication ? scenario.adjudication.state === "unresolved" : aggregateSuspect;
+        if (scenario.judge_verdict !== expectedVerdict || scenario.suspect !== expectedSuspect)
+            throw new Error(`schema v3 scenario verdict/suspect diverges from repetition aggregate for ${scenario.id}`);
+    }
+    return result;
+}
 /** Pure: return a copy with override + note applied to one scenario. */
 export function applyOverride(results, scenarioId, override, note) {
     if (override !== null && note.trim() === "") {
@@ -399,7 +599,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, judge_history: freshJudgeHistory, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -411,7 +611,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     };
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
-    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" ||
+    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
         (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
@@ -423,8 +623,8 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // Taking `suspect` from `fresh` while carrying the block therefore published a
     // record that said `state: "unresolved"` and scored as a clean SHIP. `regate`
     // did exactly that: it rebuilds verdict and suspect from the saved first-wave
-    // judge file, so a free, offline command silently resolved a disagreement in
-    // favour of shipping — inverting this module's stated invariant that an
+    // judge file, so a gate-only rewrite silently resolved a disagreement in favour
+    // of shipping — inverting this module's stated invariant that an
     // unresolved disagreement must not resolve itself.
     const unresolved = adjudication?.state === "unresolved";
     // A settled adjudication outranks a re-read of the first-wave judge file for
@@ -440,6 +640,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // corpus with a pure-noise diff.
     return {
         id,
+        ...((freshCriterionCount ?? prior?.criterion_count) === undefined ? {} : { criterion_count: freshCriterionCount ?? prior.criterion_count }),
         judge_verdict: settled ?? judge_verdict,
         judge_reason,
         suspect: suspect || unresolved,
@@ -459,6 +660,8 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         // existed.
         ...(objective ? { objective } : {}),
         ...(adjudication ? { adjudication } : {}),
+        ...((freshJudgeHistory ?? prior?.judge_history) ? { judge_history: freshJudgeHistory ?? prior.judge_history } : {}),
+        ...((freshRepJudgments ?? prior?.rep_judgments) ? { rep_judgments: freshRepJudgments ?? prior.rep_judgments } : {}),
     };
 }
 /**

package/dist/run.js

@@ -1,9 +1,9 @@
 import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { sourceHashes } from "./sources.js";
+import { sourceHashes, PROMPT_NORMALIZATION_SOURCE_KEY, PROMPT_NORMALIZATION_SOURCE_DIGEST } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
 import { NONE_ARM, seedArmDefinitions } from "./arms.js";
-import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, deliveryStatusForObservations, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -139,13 +139,20 @@ export async function runSkillModel(opts) {
         }
     });
     const flat = await runPool(tasks, opts.concurrency ?? 1);
+    const subjectInvocations = flat.flatMap(outcome => outcome.subject_invocations ?? []);
     const grouped = scenarios.map(() => []);
     flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
     const scenarioResults = scenarios.map((scenario, si) => {
         const threshold = scenario.critical
             ? effectiveThreshold(undefined, scenario)
             : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
-        return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+        const result = outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+        if (adapter.observesPrompts)
+            result.criterion_count = scenario.checklist.length;
+        if (adapter.observesPrompts && !result.rep_judgments) {
+            result.rep_judgments = grouped[si].map((outcome, repetition) => ({ repetition, judgments: [], recorded_verdict: outcome.verdict, ...(outcome.objective ? { objective: outcome.objective } : {}) }));
+        }
+        return result;
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
     const results = writeResults(runDir, {
@@ -161,8 +168,12 @@ export async function runSkillModel(opts) {
         ...(partial ? { partial: true } : {}),
         // Only the scenarios this run actually measured: a --only run must not claim
         // coverage of scenarios it skipped.
-        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
+        source_hashes: {
+            ...sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
+            ...(adapter.observesPrompts ? { [PROMPT_NORMALIZATION_SOURCE_KEY]: PROMPT_NORMALIZATION_SOURCE_DIGEST } : {}),
+        },
         scenarios: scenarioResults,
+        ...(adapter.observesPrompts ? { schema: 3, subject_invocations: subjectInvocations } : {}),
         ...(arm.name === NONE_ARM.name ? {} : {
             arm: {
                 name: arm.name,
@@ -210,11 +221,12 @@ async function runRep(scenario, rep, repCount, ctx) {
     const repField = repCount > 1 ? { rep } : {};
     const arm = ctx.arm ?? NONE_ARM;
     const skillsRoot = ctx.skillsRoot ?? dirname(ctx.skillDir);
-    // `<run-dir>` in an arm's env values is a real path (temp dirs on this box
-    // routinely contain characters a regex would treat specially), so this is a
-    // plain split/join, not a `.replace(/<run-dir>/g, …)`.
-    const armEnv = Object.keys(arm.env).length
-        ? Object.fromEntries(Object.entries(arm.env).map(([k, v]) => [k, v.split("<run-dir>").join(runDir)]))
+    // Arm paths are per-run/per-workspace values. Plain split/join avoids treating
+    // temp-path characters as regex syntax.
+    const armEnvFor = (workspace) => Object.keys(arm.env).length
+        ? Object.fromEntries(Object.entries(arm.env).map(([k, v]) => [k, v
+                .split("<run-dir>").join(runDir)
+                .split("<workspace>").join(workspace)]))
         : undefined;
     if (rep === 0) {
         log(`  ${scenario.id} (${scenario.title})${repCount > 1 ? ` ×${repCount}` : ""} …`);
@@ -228,6 +240,7 @@ async function runRep(scenario, rep, repCount, ctx) {
     // produces no diff, and writing an empty artifact there would misreport "the
     // model changed nothing" for a rep that never ran.
     let stagedDiff = null;
+    const subjectInvocations = [];
     try {
         try {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
@@ -284,6 +297,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
             // An adapter that THREW is the same class of event and takes the same retry.
             for (let attempt = 0; attempt < 2; attempt++) {
+                const observe = (prompt) => subjectInvocations.push({ scenario_id: scenario.id, repetition: rep, attempt, prompt });
                 if (attempt > 0) {
                     const why = adapterFailure ? `adapter failed (${adapterFailure})` : "empty response";
                     appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, reason: why, ...repField });
@@ -322,7 +336,8 @@ async function runRep(scenario, rep, repCount, ctx) {
                             // builds the RunReq — the arm's extensions and env (with `<run-dir>`
                             // already substituted) both must reach pi.
                             armExtensions: arm.extensions,
-                            ...(armEnv ? { armEnv } : {}),
+                            ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
+                            ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
                         });
                         transcript = r.transcript;
                         gatePrefix = r.gateFailure;
@@ -347,7 +362,8 @@ async function runRep(scenario, rep, repCount, ctx) {
                                 ...arm.extensions,
                             ],
                             eventSources: scenario.eventSources,
-                            ...(armEnv ? { armEnv } : {}),
+                            ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
+                            ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
                         };
                         if (useStructured) {
                             const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
@@ -441,6 +457,16 @@ async function runRep(scenario, rep, repCount, ctx) {
                 });
             }
         }
+        // Delivery is an objective gate on every Pi subject invocation. It is derived
+        // from captured provider payload bytes, never from mode/argv supplied by a caller.
+        let deliveryObjective;
+        if (ctx.adapter.observesPrompts) {
+            const statuses = subjectInvocations.map(observation => observation.prompt.status);
+            const status = deliveryStatusForObservations(subjectInvocations);
+            deliveryObjective = { status, assertions: [{ kind: "skill_delivered", status, detail: statuses.length === 0 ? "no model-visible prompt observation was retained" : `${statuses.length} provider request(s): ${statuses.join(", ")}` }] };
+            if (status !== "PASS")
+                gatePrefix = `objective: skill_delivered ${status.toLowerCase()} — ${deliveryObjective.assertions[0].detail}`;
+        }
         // Objective evidence is persisted for every rep, pass or fail — a failing gate
         // is exactly when someone wants to read what the model actually did.
         // `!adapterFailure`: a rep whose adapter threw never ran, so there is nothing
@@ -448,7 +474,7 @@ async function runRep(scenario, rep, repCount, ctx) {
         // which is the symptom — it names the missing artifact instead of the crash or
         // timeout that caused it, and it is that text, not the real reason, that would
         // reach the results record. The verdict is ERROR either way.
-        let objective;
+        let objective = deliveryObjective;
         if ((scenario.traceAssert || scenario.trajectoryAssert) && !adapterFailure) {
             const assertionResults = [];
             let status = "PASS";
@@ -494,6 +520,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                     trajectoryMeta = { trajectory_version: gate.event_version, events_sha256: gate.events_sha256 };
                 }
             }
+            if (deliveryObjective) {
+                if (deliveryObjective.status === "ERROR" || (deliveryObjective.status === "NOT-MEASURED" && status !== "ERROR") || (deliveryObjective.status === "FAIL" && status === "PASS"))
+                    status = deliveryObjective.status;
+                assertionResults.unshift(...deliveryObjective.assertions);
+            }
             objective = { status, ...traceMeta, ...trajectoryMeta, assertions: assertionResults };
             if (status !== "PASS") {
                 const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
@@ -507,12 +538,18 @@ async function runRep(scenario, rep, repCount, ctx) {
         let verdict;
         let reason;
         let suspect = false;
+        let judgment;
         let judgeCalls = 0;
         if (objective?.status === "ERROR") {
             verdict = "ERROR";
             reason = gatePrefix ?? "objective evidence missing";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
         }
+        else if (objective?.status === "NOT-MEASURED") {
+            verdict = "NOT-MEASURED";
+            reason = gatePrefix ?? "skill delivery was not established";
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
         else if (infrastructureFailure) {
             verdict = "ERROR";
             reason = infrastructureFailure;
@@ -536,13 +573,14 @@ async function runRep(scenario, rep, repCount, ctx) {
             });
             verdict = o.verdict;
             reason = o.reason;
-            suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
+            suspect = o.suspect;
+            judgment = o.judgment; // judgeOneRep already journaled (verdict + misfire)
             judgeCalls = 1;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
         const subject = mergeTraces(traces)?.metrics;
         return {
-            verdict, reason, suspect, objective,
+            verdict, reason, suspect, objective, judgment, subject_invocations: subjectInvocations,
             metrics: {
                 wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
                 judge_calls: judgeCalls,
@@ -577,8 +615,8 @@ export function formatScorecard(summary, lift, stability) {
     const lines = [];
     lines.push(`── ${results.skill} · ${results.harness} · ${results.model} ──`);
     for (const s of results.scenarios) {
-        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
-        const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : "?";
+        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "NOT-MEASURED" ? "NOT-MEASURED" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
+        const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : v === "NOT-MEASURED" ? "∅" : "?";
         const ov = s.override ? " (override)" : "";
         const susp = s.suspect ? " ⚠suspect" : "";
         const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
@@ -594,7 +632,18 @@ export function formatScorecard(summary, lift, stability) {
             ? "subject tokens unavailable"
             : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
         const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
-        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported) · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
+        const cost = metrics.cost_source === "subscription"
+            ? " · subscription ($0 marginal cost recorded)"
+            : metrics.cost_source === "unreported"
+                ? metrics.input_tokens !== null && metrics.input_tokens > 0
+                    ? " · WARNING: subject tokens were used but subject cost was not reported"
+                    : " · subject cost unavailable"
+                : metrics.input_tokens !== null && metrics.input_tokens > 0 && metrics.subject_cost_usd === 0
+                    ? " · WARNING: subject tokens were used but the provider reported $0 cost"
+                    : metrics.subject_cost_usd === null
+                        ? " · subject cost unavailable"
+                        : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
+        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported)${cost} · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
     }
     // Lift is a statement about a skill-delivered run (green or force). On a red run
     // the caller may still have a lift in hand (a scored run exists in the same tag),

package/dist/score.d.ts

@@ -3,7 +3,7 @@ import type { ShipBar } from "./spec.js";
  * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
  * never a pass and never silently resolved — it marks the run for a rejudge.
  */
-export type Verdict = "PASS" | "FAIL" | "ERROR" | "JUDGE-AMBIGUOUS";
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;
@@ -23,13 +23,14 @@ export interface ScoreResult {
     bSeriesFails: number;
     suspectCount: number;
     errorCount: number;
+    notMeasuredCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
- * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS and
+ * NOT-MEASURED are excluded as infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total

package/dist/score.js

@@ -11,8 +11,8 @@ export function letterFor(pct) {
 }
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
- * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS and
+ * NOT-MEASURED are excluded as infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total
@@ -27,11 +27,16 @@ export function score(verdicts, input) {
     let bSeriesFails = 0;
     let suspectCount = 0;
     let errorCount = 0;
+    let notMeasuredCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
             continue; // untrustworthy: neither pass nor fail
         }
+        if (v.verdict === "NOT-MEASURED") {
+            notMeasuredCount++;
+            continue; // known-undelivered experiment: neither behavioral pass nor fail
+        }
         if (v.verdict === "ERROR" || v.verdict === "JUDGE-AMBIGUOUS") {
             errorCount++;
             continue; // infrastructure/indeterminate: neither behavioral pass nor fail
@@ -55,7 +60,8 @@ export function score(verdicts, input) {
         (!shipBar.no_critical_fail || criticalFails === 0) &&
         bSeriesFails === 0 &&
         suspectCount === 0 &&
-        errorCount === 0;
+        errorCount === 0 &&
+        notMeasuredCount === 0;
     let note = "";
     if (!validBar) {
         note = "invalid ship bar: total/min_pass must be positive integers with min_pass <= total";
@@ -66,12 +72,15 @@ export function score(verdicts, input) {
     else if (errorCount > 0) {
         note = `${errorCount} infrastructure error${errorCount === 1 ? "" : "s"}: retry/repair evidence`;
     }
+    else if (notMeasuredCount > 0) {
+        note = `${notMeasuredCount} not measured: skill delivery was not established`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/screen.d.ts

@@ -0,0 +1,31 @@
+import { type ResultsFile } from "./results.js";
+export type ScreenClassification = "CEILING" | "FLOOR" | "INFORMATIVE" | "UNKNOWN";
+export interface ScreenRate {
+    passes: number;
+    n: number;
+}
+export interface ScreenScenario {
+    skill: string;
+    model: string;
+    id: string;
+    control: ScreenRate;
+    treatment: ScreenRate;
+    not_measured: number;
+    classification: ScreenClassification;
+}
+export interface ScreenCriterion {
+    skill: string;
+    model: string;
+    scenario_id: string;
+    criterion: number;
+    failures: number;
+    n: number;
+    fail_rate: number;
+}
+export interface ScreenReport {
+    scenarios: ScreenScenario[];
+    criteria: ScreenCriterion[];
+}
+/** Pure/offline: derive only from v3 retained invocation and vote evidence. */
+export declare function screenResults(results: ResultsFile[]): ScreenReport;
+export declare function formatScreen(report: ScreenReport): string;