@skill-harness/core 0.10.0 → 0.12.0

package/dist/results.js

@@ -4,6 +4,7 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+import { collapseVotePanel } from "./vote-panel.js";
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export function mergeScenarioMetrics(prior, fresh) {
     if (!prior)
@@ -22,11 +23,20 @@ export function mergeScenarioMetrics(prior, fresh) {
         ...(subject.cache_read_tokens === undefined ? {} : { cache_read_tokens: subject.cache_read_tokens }),
         ...(subject.cache_write_tokens === undefined ? {} : { cache_write_tokens: subject.cache_write_tokens }),
         ...(subject.subject_cost_usd === undefined ? {} : { subject_cost_usd: subject.subject_cost_usd }),
+        ...(subject.cost_source === undefined ? {} : { cost_source: subject.cost_source }),
         ...(subject.tool_calls === undefined ? {} : { tool_calls: subject.tool_calls }),
         ...(subject.delegated_children === undefined ? {} : { delegated_children: subject.delegated_children }),
         ...(subject.max_concurrency === undefined ? {} : { max_concurrency: subject.max_concurrency }),
     };
 }
+export function carryRepObjectives(fresh, prior) {
+    if (!fresh)
+        return prior;
+    return fresh.map(panel => {
+        const objective = prior?.find(candidate => candidate.repetition === panel.repetition)?.objective;
+        return objective ? { ...panel, objective } : panel;
+    });
+}
 /**
  * The run modes whose results carry a real grade: the ones where the skill under
  * test was actually delivered to the model.
@@ -87,9 +97,23 @@ export function effectiveThreshold(prevScenario, scenario) {
 function timestampSlug(iso) {
     return iso.replace(/[:.]/g, "-");
 }
-/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
-export function runDirFor(skillDir, harness, model, timestamp) {
-    return join(skillDir, "tests", "results", `${harness}-${modelSlug(model)}`, timestampSlug(timestamp));
+/**
+ * `<skillDir>/tests/results/<harness>-<model-slug>[+<arm>]/<timestamp-slug>/`
+ *
+ * The arm is part of the run's IDENTITY, not of the spec's digest. `lint` and
+ * `stability` both key on this tag, so two arms become separate lineages that can
+ * never be misread as one lineage flipping run-over-run — and no
+ * `specification.yaml` byte moves, so every committed run stays valid.
+ *
+ * `+` is deliberate and is appended OUTSIDE `modelSlug`, whose character class
+ * (`[^A-Za-z0-9._-]`) cannot emit one. So the separator can never occur inside a
+ * slug and the tag stays unambiguously splittable back into model and arm. The
+ * control arm appends nothing, so its paths are byte-identical to before arms
+ * existed.
+ */
+export function runDirFor(skillDir, harness, model, timestamp, armName) {
+    const arm = armName && armName !== "none" ? `+${armName}` : "";
+    return join(skillDir, "tests", "results", `${harness}-${modelSlug(model)}${arm}`, timestampSlug(timestamp));
 }
 /** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
 export function transcriptPath(runDir, scenarioId, mode, rep) {
@@ -145,6 +169,8 @@ function objectiveVerdict(s) {
         return undefined;
     if (s.objective.status === "ERROR")
         return "ERROR";
+    if (s.objective.status === "NOT-MEASURED")
+        return "NOT-MEASURED";
     if (s.objective.status === "FAIL")
         return "FAIL";
     return undefined;
@@ -165,8 +191,9 @@ export function finalizeResults(draft, ctx) {
         const why = draft.partial ? "partial run (--only) — not scored" : `mode=${draft.mode} (not scored)`;
         effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: why };
     }
+    const schema = draft.schema ?? (draft.subject_invocations ? 3 : 2);
     return {
-        schema: 2,
+        schema,
         // Stamped here, the single place every writer passes through, so `run`,
         // `grade`, `rescore` and the review UI's override save all record which tool
         // produced the record they leave behind.
@@ -182,15 +209,18 @@ export function finalizeResults(draft, ctx) {
         timestamp: draft.timestamp,
         label: draft.label,
         mode: draft.mode,
+        ...(draft.arm ? { arm: draft.arm } : {}),
         ...(draft.partial ? { partial: true } : {}),
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         effective_grade,
         scenarios: draft.scenarios,
+        ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),
     };
 }
 /** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
 export function writeResults(runDir, draft, ctx) {
     const results = finalizeResults(draft, ctx);
+    validateResults(results);
     mkdirSync(runDir, { recursive: true });
     writeFileSync(resultsPath(runDir), yaml.dump(results, { lineWidth: 100 }), "utf8");
     return results;
@@ -202,8 +232,8 @@ export function migrateResults(raw) {
         throw new Error("empty or invalid results.yaml");
     }
     const o = raw;
-    if (o.schema === 2)
-        return raw;
+    if (o.schema === 2 || o.schema === 3)
+        return validateResults(raw);
     const v1 = raw;
     const modeMatch = /^mode=(\w+)/.exec(v1.grade?.note ?? "");
     return {
@@ -235,6 +265,191 @@ export function readResults(runDir) {
     const text = readFileSync(resultsPath(runDir), "utf8");
     return migrateResults(yaml.load(text));
 }
+const CRITERION_RE = /^\s*(\d+)[.)]\s*\**\s*(PASS|FAIL)\b\**\s*(.*)$/gim;
+/** Parse rubric-item votes for retention only; verdict semantics remain in grade.ts. */
+export function parseCriterionVotes(raw) {
+    return [...raw.matchAll(CRITERION_RE)].map((match) => ({
+        index: Number(match[1]), verdict: match[2].toUpperCase(),
+        reason: match[3].trim().replace(/^[-—:]\s*/, ""),
+    }));
+}
+export function completeCriterionVotes(votes, expected) {
+    return Array.from({ length: expected }, (_, offset) => votes.find(vote => vote.index === offset + 1) ?? {
+        index: offset + 1, verdict: "ERROR", reason: "judge emitted no parseable vote for this criterion",
+    });
+}
+export function deliveryStatusForObservations(observations) {
+    if (observations.length === 0)
+        return "ERROR";
+    const terminalAttempt = Math.max(...observations.map(observation => observation.attempt ?? 0));
+    const terminal = observations.filter(observation => (observation.attempt ?? 0) === terminalAttempt);
+    if (terminal.some(observation => observation.prompt.status === "ERROR"))
+        return "ERROR";
+    const mechanism = terminal[0].prompt.mechanism;
+    if (terminal.some(observation => observation.prompt.mechanism !== mechanism))
+        return "ERROR";
+    const counts = terminal.map(observation => observation.prompt.contract_occurrences);
+    if (mechanism === "none")
+        return counts.every(count => count === 0) ? "PASS" : "NOT-MEASURED";
+    if (mechanism === "pi-skill") {
+        const firstDelivered = counts.indexOf(1);
+        return firstDelivered >= 0 && counts.slice(0, firstDelivered).every(count => count === 0) && counts.slice(firstDelivered).every(count => count === 1) ? "PASS" : "NOT-MEASURED";
+    }
+    return counts.every(count => count === 1) ? "PASS" : "NOT-MEASURED";
+}
+export function recomputeRecordedPanels(results) {
+    const out = [];
+    for (const scenario of results.scenarios)
+        for (const panel of scenario.rep_judgments ?? []) {
+            const clean = panel.judgments.filter(j => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
+            let verdict = panel.recorded_verdict;
+            if (clean.length === 1)
+                verdict = clean[0].verdict;
+            else if (clean.length >= 2)
+                verdict = collapseVotePanel(panel.judgments).verdict ?? "JUDGE-AMBIGUOUS";
+            out.push({ scenario_id: scenario.id, repetition: panel.repetition, verdict });
+        }
+    return out;
+}
+/** Runtime v2/v3 validator. v2 remains permissive; v3 fails closed on observation gaps. */
+export function validateResults(raw) {
+    if (!raw || typeof raw !== "object")
+        throw new Error("invalid results");
+    const result = raw;
+    if (result.schema !== 2 && result.schema !== 3)
+        throw new Error(`unsupported results schema ${String(raw.schema)}`);
+    if (!Array.isArray(result.scenarios))
+        throw new Error("results scenarios missing");
+    if (result.schema === 2)
+        return result;
+    if (!Array.isArray(result.subject_invocations))
+        throw new Error("schema v3 delivery observations missing");
+    for (const observation of result.subject_invocations) {
+        const p = observation?.prompt;
+        if (!p || !["PASS", "NOT-MEASURED", "ERROR"].includes(p.status))
+            throw new Error("delivery status missing");
+        if (p.normalization_rule !== "cwd-line-v1")
+            throw new Error("unknown prompt normalization rule");
+        if (!/^[a-f0-9]{64}$/i.test(p.raw_sha256) || !/^[a-f0-9]{64}$/i.test(p.normalized_sha256) || !/^[a-f0-9]{64}$/i.test(p.contract_sha256))
+            throw new Error("invalid prompt provenance digest");
+        if (!Number.isInteger(p.bytes) || p.bytes < 0 || !Number.isInteger(p.contract_bytes) || p.contract_bytes < 0)
+            throw new Error("invalid prompt provenance byte length");
+        if (!Number.isInteger(p.contract_occurrences) || p.contract_occurrences < 0)
+            throw new Error("invalid delivery occurrence count");
+        const expected = p.mechanism === "none" ? 0 : 1;
+        const computed = p.contract_occurrences === expected ? "PASS" : "NOT-MEASURED";
+        if (p.status !== "ERROR" && p.status !== computed)
+            throw new Error("delivery status contradicts occurrence count");
+    }
+    const scenarioIds = new Set(result.scenarios.map(scenario => scenario.id));
+    const repsByScenario = new Map(result.scenarios.map(scenario => [scenario.id, scenario.reps ?? 1]));
+    if (result.subject_invocations.some(observation => !scenarioIds.has(observation.scenario_id) || !Number.isInteger(observation.repetition) || observation.repetition < 0 || observation.repetition >= (repsByScenario.get(observation.scenario_id) ?? 0)))
+        throw new Error("schema v3 contains orphan or out-of-range subject invocation observation");
+    const recomputed = recomputeRecordedPanels(result);
+    for (const scenario of result.scenarios) {
+        const reps = scenario.reps ?? 1;
+        if (!Array.isArray(scenario.rep_judgments) || scenario.rep_judgments.length !== reps)
+            throw new Error(`schema v3 repetition judgments missing for ${scenario.id}`);
+        const panelRepetitions = scenario.rep_judgments.map(panel => panel.repetition).sort((a, b) => a - b);
+        if (panelRepetitions.some((repetition, index) => repetition !== index))
+            throw new Error(`schema v3 repetition judgments duplicate, missing, or out of range for ${scenario.id}`);
+        for (let repetition = 0; repetition < reps; repetition++) {
+            const observations = result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === repetition);
+            // A pre-provider infrastructure failure legitimately has zero requests;
+            // its per-repetition skill_delivered objective must remain explicit ERROR.
+            const attempts = [...new Set(observations.map(observation => observation.attempt ?? 0))].sort((a, b) => a - b);
+            if (attempts.some((attempt, index) => !Number.isInteger(attempt) || attempt !== index))
+                throw new Error(`delivery attempts are duplicate, missing, or out of range for ${scenario.id}#${repetition}`);
+            for (const attempt of attempts) {
+                const indexes = observations.filter(observation => (observation.attempt ?? 0) === attempt).map(observation => observation.prompt.request_index).sort((a, b) => a - b);
+                if (indexes.some((requestIndex, index) => !Number.isInteger(requestIndex) || requestIndex !== index))
+                    throw new Error(`prompt request indexes are duplicate, missing, or out of range for ${scenario.id}#${repetition} attempt ${attempt}`);
+            }
+        }
+        const repStatuses = Array.from({ length: reps }, (_, repetition) => deliveryStatusForObservations(result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === repetition)));
+        const expectedStatus = repStatuses.includes("ERROR") ? "ERROR" : repStatuses.includes("NOT-MEASURED") ? "NOT-MEASURED" : "PASS";
+        const gate = scenario.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
+        if (!gate || gate.status !== expectedStatus)
+            throw new Error(`skill_delivered gate missing or inconsistent for ${scenario.id}`);
+        if (!Number.isInteger(scenario.criterion_count) || scenario.criterion_count < 1)
+            throw new Error(`schema v3 criterion count missing for ${scenario.id}`);
+        const assertCriteria = (judgment) => {
+            if (!Array.isArray(judgment.criteria))
+                throw new Error("schema v3 criterion votes missing");
+            const indexes = judgment.criteria.map(vote => vote.index).sort((a, b) => a - b);
+            if (indexes.length !== scenario.criterion_count || indexes.some((index, offset) => index !== offset + 1))
+                throw new Error(`schema v3 criterion votes are not complete contiguous indexes for ${scenario.id}`);
+        };
+        for (const panel of scenario.rep_judgments) {
+            for (const judgment of panel.judgments)
+                assertCriteria(judgment);
+            const cleanJudgments = panel.judgments.filter(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL"));
+            const objectiveBehavioralFail = panel.objective?.status === "FAIL" && panel.recorded_verdict === "FAIL";
+            if ((panel.recorded_verdict === "PASS" || panel.recorded_verdict === "FAIL") && cleanJudgments.length === 0 && !objectiveBehavioralFail)
+                throw new Error(`recorded behavioral verdict is unsupported by a clean judgment for ${scenario.id}#${panel.repetition}`);
+            const actual = recomputed.find(x => x.scenario_id === scenario.id && x.repetition === panel.repetition);
+            if (panel.judgments.length > 0 && actual.verdict !== panel.recorded_verdict)
+                throw new Error(`recorded panel verdict diverges from recomputed votes for ${scenario.id}#${panel.repetition}`);
+            if (!panel.objective)
+                throw new Error(`schema v3 per-repetition objective missing for ${scenario.id}#${panel.repetition}`);
+            const observations = result.subject_invocations.filter(o => o.scenario_id === scenario.id && o.repetition === panel.repetition);
+            const deliveryStatus = deliveryStatusForObservations(observations);
+            const delivery = panel.objective.assertions.find(assertion => assertion.kind === "skill_delivered");
+            if (!delivery || delivery.status !== deliveryStatus)
+                throw new Error(`per-repetition skill_delivered gate missing or inconsistent for ${scenario.id}#${panel.repetition}`);
+            if (deliveryStatus === "ERROR" && panel.objective.status !== "ERROR")
+                throw new Error(`per-repetition objective weakens delivery ERROR for ${scenario.id}#${panel.repetition}`);
+            if (deliveryStatus === "NOT-MEASURED" && !["ERROR", "NOT-MEASURED"].includes(panel.objective.status))
+                throw new Error(`per-repetition objective weakens delivery NOT-MEASURED for ${scenario.id}#${panel.repetition}`);
+            const objectiveVerdict = panel.objective.status === "ERROR" ? "ERROR" : panel.objective.status === "NOT-MEASURED" ? "NOT-MEASURED" : panel.objective.status === "FAIL" ? "FAIL" : undefined;
+            if (objectiveVerdict && panel.recorded_verdict !== objectiveVerdict)
+                throw new Error(`recorded panel verdict weakens objective evidence for ${scenario.id}#${panel.repetition}`);
+        }
+        const objectiveStatuses = scenario.rep_judgments.map(panel => panel.objective.status);
+        const aggregateObjectiveStatus = objectiveStatuses.includes("ERROR") ? "ERROR" : objectiveStatuses.includes("NOT-MEASURED") ? "NOT-MEASURED" : objectiveStatuses.includes("FAIL") ? "FAIL" : "PASS";
+        if (!scenario.objective || scenario.objective.status !== aggregateObjectiveStatus)
+            throw new Error(`scenario objective diverges from per-repetition objectives for ${scenario.id}`);
+        for (const judgment of [...(scenario.judge_history ?? []), ...(scenario.adjudication?.judgments ?? [])])
+            assertCriteria(judgment);
+        let adjudicatedVerdict;
+        if (scenario.adjudication) {
+            if (!Number.isInteger(scenario.adjudication.repetition) || scenario.adjudication.repetition < 0 || scenario.adjudication.repetition >= reps)
+                throw new Error(`schema v3 adjudication repetition missing or out of range for ${scenario.id}`);
+            const collapsed = collapseVotePanel(scenario.adjudication.judgments);
+            const boundedCriticalAggregate = (scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
+            if (!boundedCriticalAggregate && (scenario.adjudication.state !== collapsed.state || scenario.adjudication.verdict !== collapsed.verdict))
+                throw new Error(`recorded adjudication state/verdict diverges from recomputed votes for ${scenario.id}`);
+            if (scenario.adjudication.state !== "unresolved")
+                adjudicatedVerdict = scenario.adjudication.verdict;
+        }
+        const panels = scenario.rep_judgments;
+        const errorCount = panels.filter(panel => panel.recorded_verdict === "ERROR").length;
+        const notMeasuredCount = panels.filter(panel => panel.recorded_verdict === "NOT-MEASURED").length;
+        const cleanPanels = panels.filter(panel => !(panel.judgments[0]?.suspect ?? false));
+        const passes = cleanPanels.filter(panel => panel.recorded_verdict === "PASS").length;
+        let aggregateVerdict;
+        let aggregateSuspect = false;
+        if (panels.length === 1) {
+            aggregateVerdict = panels[0].recorded_verdict;
+            aggregateSuspect = panels[0].judgments[0]?.suspect ?? false;
+        }
+        else if (errorCount > 0)
+            aggregateVerdict = "ERROR";
+        else if (notMeasuredCount > 0)
+            aggregateVerdict = "NOT-MEASURED";
+        else if (cleanPanels.length * 2 < panels.length) {
+            aggregateVerdict = "FAIL";
+            aggregateSuspect = true;
+        }
+        else
+            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) ? "PASS" : "FAIL";
+        const expectedVerdict = adjudicatedVerdict ?? aggregateVerdict;
+        const expectedSuspect = scenario.adjudication ? scenario.adjudication.state === "unresolved" : aggregateSuspect;
+        if (scenario.judge_verdict !== expectedVerdict || scenario.suspect !== expectedSuspect)
+            throw new Error(`schema v3 scenario verdict/suspect diverges from repetition aggregate for ${scenario.id}`);
+    }
+    return result;
+}
 /** Pure: return a copy with override + note applied to one scenario. */
 export function applyOverride(results, scenarioId, override, note) {
     if (override !== null && note.trim() === "") {
@@ -384,7 +599,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, judge_history: freshJudgeHistory, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -396,7 +611,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     };
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
-    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" ||
+    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
         (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
@@ -408,8 +623,8 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // Taking `suspect` from `fresh` while carrying the block therefore published a
     // record that said `state: "unresolved"` and scored as a clean SHIP. `regate`
     // did exactly that: it rebuilds verdict and suspect from the saved first-wave
-    // judge file, so a free, offline command silently resolved a disagreement in
-    // favour of shipping — inverting this module's stated invariant that an
+    // judge file, so a gate-only rewrite silently resolved a disagreement in favour
+    // of shipping — inverting this module's stated invariant that an
     // unresolved disagreement must not resolve itself.
     const unresolved = adjudication?.state === "unresolved";
     // A settled adjudication outranks a re-read of the first-wave judge file for
@@ -425,6 +640,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // corpus with a pure-noise diff.
     return {
         id,
+        ...((freshCriterionCount ?? prior?.criterion_count) === undefined ? {} : { criterion_count: freshCriterionCount ?? prior.criterion_count }),
         judge_verdict: settled ?? judge_verdict,
         judge_reason,
         suspect: suspect || unresolved,
@@ -444,6 +660,8 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         // existed.
         ...(objective ? { objective } : {}),
         ...(adjudication ? { adjudication } : {}),
+        ...((freshJudgeHistory ?? prior?.judge_history) ? { judge_history: freshJudgeHistory ?? prior.judge_history } : {}),
+        ...((freshRepJudgments ?? prior?.rep_judgments) ? { rep_judgments: freshRepJudgments ?? prior.rep_judgments } : {}),
     };
 }
 /**

package/dist/run.d.ts

@@ -1,5 +1,6 @@
 import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type Arm } from "./arms.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
 import { type ScenarioStability } from "./stability.js";
@@ -32,6 +33,19 @@ export interface RunOptions {
      * there) is already refused by the adapter for free.
      */
     canary?: boolean;
+    /**
+     * Take the structured path even when no scenario declares a trace or trajectory
+     * assertion. The subject half of `ScenarioMetrics` — tokens, `subject_cost_usd` —
+     * is only ever populated from traces, so without this a run records no cost or
+     * token data at all, which is why the reference corpus has none.
+     */
+    structured?: boolean;
+    /** The arm this run is measured under. Absent means the control (`none`). */
+    arm?: Arm;
+    /** Skills root — `seed_skills` paths resolve against it. Defaults to the skill's parent. */
+    skillsRoot?: string;
+    /** Injectable ambient skill root, for tests. Production reads `~/.pi/agent/skills`. */
+    ambientSkillsDir?: string;
 }
 export interface RunSummary {
     runDir: string;