@skill-harness/core 0.8.0 → 0.10.0

package/dist/regate.js

@@ -1,11 +1,12 @@
 import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { basename, join } from "node:path";
 import { parseVerdict, detectMisfire } from "./grade.js";
 import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
+import { deserializeTrajectoryEvents, evaluateTrajectoryGates } from "./trajectory-gates.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, } from "./results.js";
+import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, findTrajectoryFiles, tracePath, trajectoryPath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -88,30 +89,39 @@ export async function regateRun(opts) {
         const s = specById.get(rec.id);
         const needles = hasNeedleGates(s ?? {});
         const traceGated = Boolean(s?.traceAssert);
-        if (!s || (!needles && !traceGated))
+        const trajectoryGated = Boolean(s?.trajectoryAssert);
+        if (!s || (!needles && !traceGated && !trajectoryGated))
             continue; // nothing for regate to re-decide
         if (s.assert?.vitest || s.assert?.post_test) {
             blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
                 `no saved artifact can stand in for it, so this scenario needs a re-run`);
             continue;
         }
-        if (needles && findDiffFiles(opts.runDir, s.id, mode).length === 0) {
-            blocked.push(`${s.id}: no staged-diff artifact on disk (\`.diff.txt\` is gitignored — regate needs the run dir that produced it)`);
+        const expectedReps = rec.reps ?? 1;
+        const expectedSuffixes = expectedReps === 1 ? [undefined] : Array.from({ length: expectedReps }, (_, index) => index);
+        const complete = (paths, expectedPaths) => paths.length === expectedPaths.length && expectedPaths.every((path) => paths.includes(basename(path)));
+        if (needles && !complete(findDiffFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => diffPath(opts.runDir, s.id, mode, rep)))) {
+            blocked.push(`${s.id}: staged-diff artifacts are incomplete for ${expectedReps} recorded rep(s) — regate refuses a partial repetition set`);
             continue;
         }
         // A trace gate is only re-decidable from a saved trace. A run recorded before
         // traces existed has none, and saying so is the whole point — pretending
         // regate can answer would report a verdict derived from no evidence.
-        if (traceGated && findTraceFiles(opts.runDir, s.id, mode).length === 0) {
-            blocked.push(`${s.id}: declares assert.trace but this run saved no \`.trace.jsonl\` artifact ` +
-                `(it predates trace capture, or the trace was not persisted) — it needs a re-run`);
+        if (traceGated && !complete(findTraceFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => tracePath(opts.runDir, s.id, mode, rep)))) {
+            blocked.push(`${s.id}: assert.trace artifacts are incomplete for ${expectedReps} recorded rep(s) ` +
+                `(traces are gitignored, or capture failed) — it needs a re-run`);
+            continue;
+        }
+        if (trajectoryGated && !complete(findTrajectoryFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => trajectoryPath(opts.runDir, s.id, mode, rep)))) {
+            blocked.push(`${s.id}: assert.trajectory artifacts are incomplete for ${expectedReps} recorded rep(s) ` +
+                `(events are gitignored, or required native evidence was missing) — it needs a re-run`);
             continue;
         }
         targets.push(s);
     }
     if (targets.length === 0) {
         throw new Error(`nothing to regate in ${opts.runDir}` +
-            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff_contains/diff_excludes or assert.trace"));
+            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff_contains/diff_excludes, assert.trace, or assert.trajectory"));
     }
     const changes = [];
     let judgeCalls = 0;
@@ -124,17 +134,21 @@ export async function regateRun(opts) {
         }
         const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
         const traceFiles = findTraceFiles(opts.runDir, scenario.id, mode);
-        // Reps come from whichever artifact this scenario actually has. A trace-only
-        // scenario has no `.diff.txt` at all, so iterating diffs would silently
-        // regate nothing and report success.
+        const eventFiles = findTrajectoryFiles(opts.runDir, scenario.id, mode);
+        // Reps come from whichever artifact this scenario actually has. A trajectory-only
+        // scenario has neither diff nor trace, so falling back only once would regate
+        // nothing and report success.
         const repKeys = diffFiles.length > 0
             ? diffFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: f }))
-            : traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
+            : traceFiles.length > 0
+                ? traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }))
+                : eventFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
         const outcomes = [];
         // Per scenario, not run-wide: with several regated scenarios, a global counter
         // would report every change as "re-judged" because some other scenario was.
         let judgedHere = 0;
         let gateFailedHere = false;
+        let gateErroredHere = false;
         for (const { rep, diffFile } of repKeys) {
             const diff = diffFile ? readFileSync(join(opts.runDir, diffFile), "utf8") : "";
             const needleGate = diffFile ? evaluateNeedleGates(scenario, diff) : { lines: [], failure: null };
@@ -166,14 +180,69 @@ export async function regateRun(opts) {
                 }
                 else {
                     const g = evaluateTraceGates(scenario.traceAssert, merged);
-                    objective = { status: g.status, trace_version: merged.trace_version, trace_sha256: merged.trace_sha256, assertions: g.assertions };
-                    if (g.status === "FAIL" || g.status === "ERROR") {
-                        const bad = g.assertions.filter((x) => x.status === g.status).map((x) => x.detail);
+                    const expectedHash = rec.objective?.rep_trace_sha256?.[rep ?? 0] ?? (rec.reps === undefined ? rec.objective?.trace_sha256 : undefined);
+                    const digestMismatch = expectedHash !== undefined && expectedHash !== merged.trace_sha256;
+                    const integrityAssertion = digestMismatch
+                        ? [{ kind: "trace_evidence", status: "ERROR", detail: "saved trace no longer matches the hash recorded by the run" }]
+                        : [];
+                    objective = { status: digestMismatch ? "ERROR" : g.status, trace_version: merged.trace_version, trace_sha256: merged.trace_sha256, assertions: [...integrityAssertion, ...g.assertions] };
+                    if (g.status === "FAIL" || g.status === "ERROR" || digestMismatch) {
+                        const bad = [...integrityAssertion.map((x) => x.detail), ...g.assertions.filter((x) => x.status === g.status && g.status !== "PASS").map((x) => x.detail)];
                         traceFailure = `objective: ${bad.join("; ")}`;
                     }
                 }
             }
-            const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure };
+            // Adapter-neutral trajectory gate, replayed from the saved normalized
+            // event artifact. Native ledgers are deliberately not re-read here: the
+            // artifact is the immutable evidence this run actually captured.
+            let trajectoryFailure = null;
+            if (scenario.trajectoryAssert) {
+                const ep = trajectoryPath(opts.runDir, scenario.id, mode, rep);
+                const events = existsSync(ep) ? deserializeTrajectoryEvents(readFileSync(ep, "utf8")) : null;
+                // The normalized artifact contains only events that were successfully captured.
+                // Source-read/normalization failures are persisted on the original objective;
+                // dropping them here would let a partial ledger become PASS on replay.
+                const priorEvidenceErrors = (rec.objective?.assertions ?? [])
+                    .filter((result) => result.kind === "trajectory_evidence" && result.status === "ERROR");
+                if (!events) {
+                    trajectoryFailure = "objective: saved normalized events are missing, malformed, or from an unsupported version";
+                    objective = {
+                        ...(objective ?? { assertions: [] }),
+                        status: "ERROR",
+                        assertions: [...(objective?.assertions ?? []), { kind: "trajectory_evidence", status: "ERROR", detail: trajectoryFailure }],
+                    };
+                }
+                else {
+                    const g = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
+                    const expectedHash = rec.objective?.rep_events_sha256?.[rep ?? 0] ?? (rec.reps === undefined ? rec.objective?.events_sha256 : undefined);
+                    const digestMismatch = expectedHash !== undefined && expectedHash !== g.events_sha256;
+                    const integrityAssertion = digestMismatch
+                        ? [{ kind: "trajectory_evidence", status: "ERROR", detail: "saved normalized events no longer match the hash recorded by the run" }]
+                        : [];
+                    const priorStatus = objective?.status ?? "PASS";
+                    const status = priorStatus === "ERROR" || g.status === "ERROR" || priorEvidenceErrors.length || digestMismatch
+                        ? "ERROR"
+                        : priorStatus === "FAIL" || g.status === "FAIL"
+                            ? "FAIL"
+                            : "PASS";
+                    objective = {
+                        ...(objective ?? {}),
+                        status,
+                        trajectory_version: g.event_version,
+                        events_sha256: g.events_sha256,
+                        assertions: [...(objective?.assertions ?? []), ...priorEvidenceErrors, ...integrityAssertion, ...g.assertions],
+                    };
+                    if (g.status !== "PASS" || priorEvidenceErrors.length || digestMismatch) {
+                        const details = [
+                            ...priorEvidenceErrors.map((result) => result.detail),
+                            ...integrityAssertion.map((result) => result.detail),
+                            ...g.assertions.filter((result) => result.status === g.status && g.status !== "PASS").map((result) => result.detail),
+                        ];
+                        trajectoryFailure = `objective: ${details.join("; ")}`;
+                    }
+                }
+            }
+            const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure ?? trajectoryFailure };
             const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
             // Two sources, because the two gate kinds record their prior state
@@ -189,8 +258,11 @@ export async function regateRun(opts) {
             if (existsSync(tPath))
                 rewriteTranscript(tPath, gate.lines);
             if (gate.failure) {
-                gateFailedHere = true;
-                outcomes.push({ verdict: "FAIL", reason: gate.failure, suspect: false, objective });
+                if (objective?.status === "ERROR")
+                    gateErroredHere = true;
+                else
+                    gateFailedHere = true;
+                outcomes.push({ verdict: objective?.status === "ERROR" ? "ERROR" : "FAIL", reason: gate.failure, suspect: false, objective });
                 continue;
             }
             if (!oldGateFailed) {
@@ -205,13 +277,14 @@ export async function regateRun(opts) {
             outcomes.push({ ...(await judgeOneRep({
                     runDir: opts.runDir, spec: opts.spec, scenario, transcript,
                     adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                    mode, rep, now,
+                    mode, rep, now, rejudge: true,
                 })), objective });
             judgeCalls++;
             judgedHere++;
         }
         const threshold = effectiveThreshold(rec, scenario);
         const next = outcomesToResult(scenario.id, outcomes, outcomes.length, threshold);
+        next.metrics = mergeScenarioMetrics(rec.metrics, next.metrics);
         // Overrides and their notes survive: a regate re-decides the gate, and an author
         // override is a statement about the judge, not about the needle.
         // `regate` re-evaluates gates from saved artifacts and asks no judge anything:
@@ -222,7 +295,7 @@ export async function regateRun(opts) {
         if (to !== rec.judge_verdict) {
             changes.push({
                 id: scenario.id, from: rec.judge_verdict, to,
-                gate: gateFailedHere ? "fail" : "pass",
+                gate: gateErroredHere ? "error" : gateFailedHere ? "fail" : "pass",
                 judged: judgedHere > 0,
             });
         }

package/dist/regrade.d.ts

@@ -32,6 +32,8 @@ export interface RegradeOptions {
      * with "nothing to re-grade", which is how ten scorable runs stayed ungraded.
      */
     mode?: string;
+    /** Recorded repetition count; when supplied, missing/extra rep artifacts fail before judge calls. */
+    expectedReps?: number;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -46,6 +48,8 @@ export declare function judgeOneRep(opts: {
     mode: string;
     rep: number | undefined;
     now: () => string;
+    /** True when this call revisits a saved subject transcript. */
+    rejudge?: boolean;
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no

package/dist/regrade.js

@@ -1,7 +1,7 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, } from "./results.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
@@ -36,6 +36,7 @@ export function refreshRubricHashes(recorded, spec, judgedIds) {
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
+    const startedAt = performance.now();
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
@@ -43,7 +44,14 @@ export async function judgeOneRep(opts) {
     appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
     if (g.suspect)
         appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
-    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
+    return {
+        verdict: g.verdict, reason: g.reason, suspect: g.suspect,
+        metrics: {
+            wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
+            judge_calls: 1,
+            judge_rejudge_calls: opts.rejudge ? 1 : 0,
+        },
+    };
 }
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -58,14 +66,20 @@ export async function regradeScenario(opts) {
     const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
     if (files.length === 0)
         throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
-    const repCount = files.length;
+    const expected = opts.expectedReps ?? files.length;
+    const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
+    const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
+    if (files.length !== expected || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
+        throw new Error(`${opts.scenario.id}: transcript artifacts are incomplete for ${expected} recorded rep(s) — re-run instead of grading a smaller repetition set`);
+    }
+    const repCount = expected;
     const outcomes = [];
     for (const file of files) {
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
-            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now,
+            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
         }));
     }
     return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
@@ -120,7 +134,15 @@ export async function regradeRun(opts) {
             return prev;
         }
     }
-    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, mode).length === 0);
+    const completeTranscripts = (record) => {
+        const expected = record.reps ?? 1;
+        const files = findTranscriptFiles(runDir, record.id, mode);
+        const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
+        const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
+        return files.length === expected && JSON.stringify(actualIndices) === JSON.stringify(expectedIndices);
+    };
+    const recordedById = new Map(recorded.map((record) => [record.id, record]));
+    const missing = targets.filter((id) => !specById.has(id) || !completeTranscripts(recordedById.get(id)));
     if (missing.length === targets.length) {
         throw new Error(`no ${mode} transcripts in ${runDir} — nothing to re-grade`);
     }
@@ -140,8 +162,10 @@ export async function regradeRun(opts) {
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
             runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
+            expectedReps: prevScenario?.reps ?? 1,
         });
         const carry = overrides.get(id);
+        rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
         // `grade` re-judges the saved transcript. It does not re-evaluate trace gates
         // (that is `regate`), so `objective` still describes this run; and it replaced
         // the judgments a prior adjudication described, so that panel must go.

package/dist/report.d.ts

@@ -1,6 +1,7 @@
 import { type ShipBar } from "./spec.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
+import { type AggregateMetrics } from "./comparison.js";
 export interface RunColumn {
     index: number;
     label: string;
@@ -8,8 +9,10 @@ export interface RunColumn {
     runDir: string;
     timestamp: string;
     mode: string;
+    partial: boolean;
     grade: ResultsFile["effective_grade"];
     judge: ResultsFile["judge"];
+    metrics: AggregateMetrics;
     cells: Record<string, {
         judge_verdict: string;
         judge_reason: string;
@@ -30,6 +33,7 @@ export interface RunColumn {
         passes?: number;
         clean?: number;
         flakiness?: number;
+        metrics?: import("./results.js").ScenarioMetrics;
         override: string | null;
         note: string;
         /**
@@ -92,11 +96,13 @@ export declare function publicView(data: ReportData): {
         tag: string;
         timestamp: string;
         mode: string;
+        partial: boolean;
         grade: import("./results.js").GradeSummary;
         judge: {
             provider: string;
             model: string;
         };
+        metrics: AggregateMetrics;
         cells: Record<string, {
             judge_verdict: string;
             judge_reason: string;
@@ -117,6 +123,7 @@ export declare function publicView(data: ReportData): {
             passes?: number;
             clean?: number;
             flakiness?: number;
+            metrics?: import("./results.js").ScenarioMetrics;
             override: string | null;
             note: string;
             /**

package/dist/report.js

@@ -4,6 +4,7 @@ import { loadSpec } from "./spec.js";
 import { readResults } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
 import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
+import { aggregateMetrics } from "./comparison.js";
 /** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
 function latestRunDir(tagDir) {
     if (!statSync(tagDir).isDirectory())
@@ -83,6 +84,7 @@ export function collectReport(skillDir) {
                     passes: s.passes,
                     clean: s.clean,
                     flakiness: s.flakiness,
+                    metrics: s.metrics,
                     override: s.override,
                     note: s.note,
                 };
@@ -105,8 +107,10 @@ export function collectReport(skillDir) {
                 runDir,
                 timestamp: r.timestamp,
                 mode: r.mode,
+                partial: r.partial === true,
                 grade: r.effective_grade,
                 judge: r.judge,
+                metrics: aggregateMetrics(r.scenarios),
                 cells,
                 ...(lift ? { lift, liftHeadline: liftHeadline(lift) } : {}),
             });
@@ -127,8 +131,10 @@ export function publicView(data) {
             tag: c.tag,
             timestamp: c.timestamp,
             mode: c.mode,
+            partial: c.partial,
             grade: c.grade,
             judge: c.judge,
+            metrics: c.metrics,
             cells: c.cells,
             ...(c.lift ? { lift: c.lift, liftHeadline: c.liftHeadline } : {}),
         })),

package/dist/reps.d.ts

@@ -1,12 +1,19 @@
 import type { Verdict } from "./score.js";
 import type { ScenarioResult, ObjectiveResult } from "./results.js";
+import type { TraceMetrics } from "./capture-trace-types.js";
 /** One rep's outcome (subject run + judge). */
 export interface RepOutcome {
     verdict: Verdict;
     reason: string;
     suspect: boolean;
-    /** Present only when the scenario declared `assert.trace`. */
+    /** Present only when the scenario declared structured objective assertions. */
     objective?: ObjectiveResult;
+    metrics?: {
+        wall_time_ms: number;
+        judge_calls: number;
+        judge_rejudge_calls: number;
+        subject?: TraceMetrics;
+    };
 }
 /**
  * Collapse per-rep objective results.

package/dist/reps.js

@@ -12,13 +12,16 @@ export function aggregateObjective(outcomes) {
     const present = outcomes.map((o) => o.objective).filter((o) => o !== undefined);
     if (present.length === 0)
         return undefined;
-    const errored = present.find((o) => o.status === "ERROR");
-    if (errored)
-        return errored;
-    const failed = present.find((o) => o.status === "FAIL");
-    if (failed)
-        return failed;
-    return present[0];
+    const picked = present.find((o) => o.status === "ERROR") ?? present.find((o) => o.status === "FAIL") ?? present[0];
+    if (present.length === 1)
+        return picked;
+    const eventHashes = present.map((objective) => objective.events_sha256);
+    const traceHashes = present.map((objective) => objective.trace_sha256);
+    return {
+        ...picked,
+        ...(eventHashes.every((hash) => typeof hash === "string") ? { rep_events_sha256: eventHashes } : {}),
+        ...(traceHashes.every((hash) => typeof hash === "string") ? { rep_trace_sha256: traceHashes } : {}),
+    };
 }
 /**
  * Collapse N rep outcomes into one scenario verdict. A rep is "clean" when its
@@ -31,14 +34,17 @@ export function aggregateReps(outcomes, threshold) {
     const reps = outcomes.length;
     const clean = outcomes.filter((o) => !o.suspect);
     const passes = clean.filter((o) => o.verdict === "PASS").length;
+    const errored = outcomes.filter((o) => o.verdict === "ERROR").length;
+    // Infrastructure/judge/tooling errors are not behavioral failures, and they
+    // cannot be voted into a pass by successful siblings. Preserve the third
+    // state so a release blocks for missing evidence without blaming the skill.
+    if (errored > 0) {
+        return { verdict: "ERROR", reason: `${errored}/${reps} reps errored — infrastructure, not behavior`, passes, reps, clean: clean.length, flakiness: 0, suspect: false };
+    }
     if (clean.length * 2 < reps) {
         // majority of reps misfired → untrustworthy
         return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
     }
-    const errored = clean.filter((o) => o.verdict === "ERROR").length;
-    if (clean.length > 0 && errored === clean.length) {
-        return { verdict: "ERROR", reason: `${errored}/${reps} reps errored`, passes: 0, reps, clean: clean.length, flakiness: 0, suspect: false };
-    }
     const passRate = passes / clean.length;
     const verdict = passRate >= threshold ? "PASS" : "FAIL";
     const flakiness = 1 - Math.abs(2 * passRate - 1);
@@ -57,15 +63,43 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     // produce a result byte-identical to one from before this field existed.
     const objective = aggregateObjective(outcomes);
     const objectiveField = objective ? { objective } : {};
+    const metrics = aggregateMetrics(outcomes);
+    const metricsField = metrics ? { metrics } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, override: null, note: "", ...objectiveField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, override: null, note: "", ...objectiveField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, override: null, note: "", ...objectiveField,
+        pass_threshold: threshold, ...metricsField, override: null, note: "", ...objectiveField,
+    };
+}
+function aggregateMetrics(outcomes) {
+    const present = outcomes.map((outcome) => outcome.metrics).filter((metrics) => metrics !== undefined);
+    if (present.length === 0)
+        return undefined;
+    const subjects = present.map((metrics) => metrics.subject).filter((metrics) => metrics !== undefined);
+    const base = {
+        wall_time_ms: present.reduce((sum, metrics) => sum + metrics.wall_time_ms, 0),
+        judge_calls: present.reduce((sum, metrics) => sum + metrics.judge_calls, 0),
+        judge_rejudge_calls: present.reduce((sum, metrics) => sum + metrics.judge_rejudge_calls, 0),
+        subject_metrics_reps: subjects.length,
+        total_reps: outcomes.length,
+    };
+    if (subjects.length === 0)
+        return base;
+    return {
+        ...base,
+        input_tokens: subjects.reduce((sum, metrics) => sum + metrics.input_tokens, 0),
+        output_tokens: subjects.reduce((sum, metrics) => sum + metrics.output_tokens, 0),
+        cache_read_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_read_tokens, 0),
+        cache_write_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_write_tokens, 0),
+        subject_cost_usd: subjects.reduce((sum, metrics) => sum + metrics.cost_usd, 0),
+        tool_calls: subjects.reduce((sum, metrics) => sum + metrics.tool_calls, 0),
+        delegated_children: subjects.reduce((sum, metrics) => sum + metrics.delegated_children, 0),
+        max_concurrency: Math.max(...subjects.map((metrics) => metrics.max_concurrency)),
     };
 }
 //# sourceMappingURL=reps.js.map

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults, scoreContextFor, rebuildScenarioResult } from "./results.js";
+import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, effectiveThreshold } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -56,7 +56,7 @@ export function rescoreRun(opts) {
             return s;
         if (s.clean === 0)
             return s;
-        const toThreshold = scenario.passThreshold ?? 0.5;
+        const toThreshold = effectiveThreshold(undefined, scenario);
         const fromThreshold = s.pass_threshold ?? toThreshold;
         if (toThreshold === fromThreshold)
             return s;

package/dist/results.d.ts

@@ -2,6 +2,23 @@ import { type ModelRef } from "./adapters/types.js";
 import { type ScenarioVerdict } from "./score.js";
 import type { Verdict } from "./score.js";
 import type { ShipBar, Scenario } from "./spec.js";
+export interface ScenarioMetrics {
+    wall_time_ms: number;
+    judge_calls: number;
+    judge_rejudge_calls: number;
+    subject_metrics_reps: number;
+    total_reps: number;
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_read_tokens?: number;
+    cache_write_tokens?: number;
+    subject_cost_usd?: number;
+    tool_calls?: number;
+    delegated_children?: number;
+    max_concurrency?: number;
+}
+/** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
+export declare function mergeScenarioMetrics(prior: ScenarioMetrics | undefined, fresh: ScenarioMetrics | undefined): ScenarioMetrics | undefined;
 export interface ScenarioResult {
     id: string;
     judge_verdict: Verdict;
@@ -14,6 +31,8 @@ export interface ScenarioResult {
     clean?: number;
     flakiness?: number;
     pass_threshold?: number;
+    /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
+    metrics?: ScenarioMetrics;
     /**
      * Objective trace-gate evidence. ADDITIVE and optional.
      *
@@ -59,6 +78,11 @@ export interface ObjectiveResult {
     status: "PASS" | "FAIL" | "ERROR";
     trace_version?: number;
     trace_sha256?: string;
+    trajectory_version?: string;
+    events_sha256?: string;
+    /** Per-repetition hashes retained when an aggregate has more than one rep. */
+    rep_events_sha256?: string[];
+    rep_trace_sha256?: string[];
     assertions: {
         kind: string;
         status: "PASS" | "FAIL" | "ERROR";
@@ -327,6 +351,10 @@ export declare function rebuildScenarioResult(fresh: ScenarioResult, prior: Scen
  * traces append without a wrapper.
  */
 export declare function tracePath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** Saved adapter-neutral workflow events for one scenario rep. */
+export declare function trajectoryPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's normalized-event files, sorted (plain first, then numeric rep). */
+export declare function findTrajectoryFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */