@skill-harness/core 0.8.0 → 0.10.0

package/dist/results.js

@@ -4,6 +4,29 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+/** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
+export function mergeScenarioMetrics(prior, fresh) {
+    if (!prior)
+        return fresh;
+    if (!fresh)
+        return prior;
+    const subject = prior.subject_metrics_reps > 0 ? prior : fresh;
+    return {
+        wall_time_ms: prior.wall_time_ms + fresh.wall_time_ms,
+        judge_calls: prior.judge_calls + fresh.judge_calls,
+        judge_rejudge_calls: prior.judge_rejudge_calls + fresh.judge_rejudge_calls,
+        subject_metrics_reps: subject.subject_metrics_reps,
+        total_reps: prior.total_reps,
+        ...(subject.input_tokens === undefined ? {} : { input_tokens: subject.input_tokens }),
+        ...(subject.output_tokens === undefined ? {} : { output_tokens: subject.output_tokens }),
+        ...(subject.cache_read_tokens === undefined ? {} : { cache_read_tokens: subject.cache_read_tokens }),
+        ...(subject.cache_write_tokens === undefined ? {} : { cache_write_tokens: subject.cache_write_tokens }),
+        ...(subject.subject_cost_usd === undefined ? {} : { subject_cost_usd: subject.subject_cost_usd }),
+        ...(subject.tool_calls === undefined ? {} : { tool_calls: subject.tool_calls }),
+        ...(subject.delegated_children === undefined ? {} : { delegated_children: subject.delegated_children }),
+        ...(subject.max_concurrency === undefined ? {} : { max_concurrency: subject.max_concurrency }),
+    };
+}
 /**
  * The run modes whose results carry a real grade: the ones where the skill under
  * test was actually delivered to the model.
@@ -53,6 +76,11 @@ export function scoreContextFor(run, spec) {
 }
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export function effectiveThreshold(prevScenario, scenario) {
+    // Critical includes destructive/security/capability/data-loss AND right-sizing
+    // counterexamples. Every clean repetition must pass; an aggregate majority may
+    // not hide one release-blocking failure.
+    if (scenario.critical)
+        return 1;
     return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
 }
 /** Slugify an ISO timestamp into a filesystem-safe directory name. */
@@ -252,7 +280,7 @@ export function ensureResultsGitignore(resultsRoot) {
 // so `repIndexOf` returned null for traces and `regate` looked for an unsuffixed
 // path that does not exist on a multi-rep run — reporting "trace missing" for
 // traces sitting on disk.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?(?:txt|trace\.jsonl)$/;
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:(?:judge|diff)\.txt|(?:trace|events)\.jsonl|txt)$/;
 /** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
@@ -356,7 +384,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -367,7 +395,14 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         return p === "fresh" ? freshValue : priorValue;
     };
     const objective = pick(policy.objective, freshObjective, prior?.objective);
-    const adjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" ||
+        (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+    // A settled cell-level panel cannot overrule newly evaluated objective gates,
+    // nor can one transcript erase a failed repetition under an all-clean policy.
+    const adjudication = conflictsWithFreshEvidence && pickedAdjudication
+        ? { ...pickedAdjudication, state: "unresolved", verdict: undefined }
+        : pickedAdjudication;
     // `unresolved` is carried by the `suspect` flag and by nothing else — the
     // adjudication block records WHY, but `suspect` is what the ship bar reads.
     // Taking `suspect` from `fresh` while carrying the block therefore published a
@@ -382,7 +417,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // `<id>.judge.txt` would revert a `confirmed`/`tie_broken` verdict that two or
     // three judges settled — leaving `adjudication.verdict` on the record
     // contradicting the `judge_verdict` beside it.
-    const settled = policy.adjudication === "carry" ? adjudication?.verdict : undefined;
+    const settled = policy.adjudication === "carry" && !conflictsWithFreshEvidence ? adjudication?.verdict : undefined;
     // Field ORDER matches `outcomesToResult`, the writer that produces a run's
     // results.yaml in the first place. It is not cosmetic: `results.yaml` is a
     // committed file, and emitting the same fields in a different order made every
@@ -400,6 +435,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(clean === undefined ? {} : { clean }),
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
         // The author owns the verdict; a re-measurement never discards their call.
         override: prior?.override ?? null,
         note: prior?.note ?? "",
@@ -421,6 +457,21 @@ export function tracePath(runDir, scenarioId, mode, rep) {
     const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
     return join(runDir, `${base}.trace.jsonl`);
 }
+/** Saved adapter-neutral workflow events for one scenario rep. */
+export function trajectoryPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.events.jsonl`);
+}
+/** A scenario's normalized-event files, sorted (plain first, then numeric rep). */
+export function findTrajectoryFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.events\\.jsonl$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.events\\.jsonl$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export function findDiffFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -469,6 +520,7 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         // override whose justification was gitignored, and left `regate` with nothing
         // to re-evaluate on the one cell a human had disputed.
         ...findTraceFiles(runDir, scenarioId),
+        ...findTrajectoryFiles(runDir, scenarioId),
     ];
     if (files.length === 0)
         return;

package/dist/run.js

@@ -2,18 +2,20 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
 import { evaluateTraceGates } from "./trace-gates.js";
+import { evaluateTrajectoryGates, serializeTrajectoryEvents, } from "./trajectory-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
+import { aggregateMetrics } from "./comparison.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -107,7 +109,9 @@ export async function runSkillModel(opts) {
     const grouped = scenarios.map(() => []);
     flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
     const scenarioResults = scenarios.map((scenario, si) => {
-        const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
+        const threshold = scenario.critical
+            ? effectiveThreshold(undefined, scenario)
+            : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
@@ -152,6 +156,7 @@ export function hasEmptyAssistantTurn(transcript) {
 }
 /** Run ONE rep of a scenario in its own isolated workspace. */
 async function runRep(scenario, rep, repCount, ctx) {
+    const startedAt = performance.now();
     const { spec, judge, mode, runDir, now, log } = ctx;
     const repField = repCount > 1 ? { rep } : {};
     if (rep === 0) {
@@ -161,6 +166,7 @@ async function runRep(scenario, rep, repCount, ctx) {
     let ws = null;
     let transcript = "";
     let gatePrefix = null;
+    let infrastructureFailure = null;
     // Null until a seeded rep actually reaches its gates: a workspace-setup failure
     // produces no diff, and writing an empty artifact there would misreport "the
     // model changed nothing" for a rep that never ran.
@@ -170,12 +176,16 @@ async function runRep(scenario, rep, repCount, ctx) {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
         }
         catch (e) {
-            // A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
+            // A setup failure means the scenario never executed. It blocks release as
+            // infrastructure ERROR; it is not evidence of subject behavior.
             gatePrefix = e instanceof Error ? e.message : String(e);
+            infrastructureFailure = gatePrefix;
             transcript = `[workspace setup failed] ${gatePrefix}`;
         }
         let noResponse = false;
         let traces = [];
+        let events = [];
+        let eventErrors = [];
         let unobservablePaths = false;
         // The pre-run state, captured AFTER `createWorkspace` has applied the
         // fixture's `_staged/` and `_uncommitted/` trees. Those land after the
@@ -201,12 +211,15 @@ async function runRep(scenario, rep, repCount, ctx) {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
                         specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
-                        trace: scenario.traceAssert ? { scenarioId: scenario.id, rep } : undefined,
+                        trace: scenario.traceAssert || scenario.trajectoryAssert ? { scenarioId: scenario.id, rep } : undefined,
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
+                    infrastructureFailure = r.gateError;
                     stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
                     traces = r.traces;
+                    events = r.events;
+                    eventErrors = r.eventErrors;
                 }
                 else {
                     const req = {
@@ -217,18 +230,21 @@ async function runRep(scenario, rep, repCount, ctx) {
                             : undefined,
                         // Absolute before it reaches a child process running in a neutral cwd.
                         extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
+                        eventSources: scenario.eventSources,
                     };
-                    if (scenario.traceAssert) {
+                    if (scenario.traceAssert || scenario.trajectoryAssert) {
                         // Missing required evidence is ERROR, never a silent fallback to the
                         // unstructured path: a gate with nothing to read must not look like a
                         // gate that passed.
                         if (!ctx.adapter.runStructured) {
-                            throw new Error(`scenario \`${scenario.id}\` declares \`assert.trace\`, but the \`${ctx.adapter.name}\` adapter` +
-                                ` cannot produce execution traces — the gate would have no evidence to read.`);
+                            throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
+                                ` cannot produce execution traces/events — the gate would have no evidence to read.`);
                         }
                         const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
                         transcript = structured.transcript;
                         traces = structured.traces;
+                        events = structured.events ?? [];
+                        eventErrors = structured.eventErrors ?? [];
                     }
                     else {
                         transcript = await ctx.adapter.run(req);
@@ -279,34 +295,55 @@ async function runRep(scenario, rep, repCount, ctx) {
         // Objective evidence is persisted for every rep, pass or fail — a failing gate
         // is exactly when someone wants to read what the model actually did.
         let objective;
-        if (scenario.traceAssert) {
-            if (traces.length > 0) {
-                writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
-            }
-            const merged = mergeTraces(traces);
-            if (unobservablePaths) {
-                gatePrefix = "objective: workspace changes could not be observed — `unchanged_paths` has no evidence to check";
-                objective = { status: "ERROR", assertions: [] };
-            }
-            else if (merged === null) {
-                // Declared a gate, produced no trace: that is broken infrastructure, and
-                // grading it either way would be inventing a result.
-                gatePrefix = "objective: no execution trace was produced — cannot evaluate assert.trace";
-                objective = { status: "ERROR", assertions: [] };
+        if (scenario.traceAssert || scenario.trajectoryAssert) {
+            const assertionResults = [];
+            let status = "PASS";
+            let traceMeta = {};
+            let trajectoryMeta = {};
+            if (scenario.traceAssert) {
+                if (traces.length > 0) {
+                    writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+                }
+                const merged = mergeTraces(traces);
+                if (unobservablePaths) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "unchanged_path", status: "ERROR", detail: "workspace changes could not be observed" });
+                }
+                else if (merged === null) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: "no execution trace was produced" });
+                }
+                else {
+                    const gate = evaluateTraceGates(scenario.traceAssert, merged);
+                    status = gate.status;
+                    assertionResults.push(...gate.assertions);
+                    traceMeta = { trace_version: merged.trace_version, trace_sha256: merged.trace_sha256 };
+                }
             }
-            else {
-                const gate = evaluateTraceGates(scenario.traceAssert, merged);
-                objective = {
-                    status: gate.status,
-                    trace_version: merged.trace_version,
-                    trace_sha256: merged.trace_sha256,
-                    assertions: gate.assertions,
-                };
-                if (gate.status === "FAIL") {
-                    // Set the same gatePrefix the seeded gates use, so a trace failure
-                    // short-circuits the judge through the path that already exists.
-                    gatePrefix = `objective: ${gate.assertions.filter((x) => x.status === "FAIL").map((x) => x.detail).join("; ")}`;
+            if (scenario.trajectoryAssert) {
+                if (events.length > 0) {
+                    writeFileSync(trajectoryPath(runDir, scenario.id, mode, repSuffix), serializeTrajectoryEvents(events), "utf8");
+                }
+                if (eventErrors.length > 0) {
+                    status = "ERROR";
+                    assertionResults.push(...eventErrors.map((detail) => ({ kind: "trajectory_evidence", status: "ERROR", detail })));
                 }
+                else if (events.length === 0) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "trajectory_evidence", status: "ERROR", detail: "no normalized workflow events were produced" });
+                }
+                else {
+                    const gate = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
+                    if (gate.status === "ERROR" || (gate.status === "FAIL" && status === "PASS"))
+                        status = gate.status;
+                    assertionResults.push(...gate.assertions);
+                    trajectoryMeta = { trajectory_version: gate.event_version, events_sha256: gate.events_sha256 };
+                }
+            }
+            objective = { status, ...traceMeta, ...trajectoryMeta, assertions: assertionResults };
+            if (status !== "PASS") {
+                const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
+                gatePrefix = `objective: ${details.join("; ") || "structured evidence could not be evaluated"}`;
             }
             appendJournal(runDir, {
                 event: "objective-result", ts: now(), id: scenario.id,
@@ -316,11 +353,17 @@ async function runRep(scenario, rep, repCount, ctx) {
         let verdict;
         let reason;
         let suspect = false;
+        let judgeCalls = 0;
         if (objective?.status === "ERROR") {
             verdict = "ERROR";
             reason = gatePrefix ?? "objective evidence missing";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
         }
+        else if (infrastructureFailure) {
+            verdict = "ERROR";
+            reason = infrastructureFailure;
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
         else if (noResponse) {
             verdict = "ERROR";
             reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
@@ -340,9 +383,19 @@ async function runRep(scenario, rep, repCount, ctx) {
             verdict = o.verdict;
             reason = o.reason;
             suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
+            judgeCalls = 1;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        return { verdict, reason, suspect, objective };
+        const subject = mergeTraces(traces)?.metrics;
+        return {
+            verdict, reason, suspect, objective,
+            metrics: {
+                wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
+                judge_calls: judgeCalls,
+                judge_rejudge_calls: 0,
+                ...(subject ? { subject } : {}),
+            },
+        };
     }
     finally {
         ws?.cleanup();
@@ -370,7 +423,7 @@ export function formatScorecard(summary, lift, stability) {
     const lines = [];
     lines.push(`── ${results.skill} · ${results.harness} · ${results.model} ──`);
     for (const s of results.scenarios) {
-        const v = s.override ?? s.judge_verdict;
+        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
         const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : "?";
         const ov = s.override ? " (override)" : "";
         const susp = s.suspect ? " ⚠suspect" : "";
@@ -381,6 +434,14 @@ export function formatScorecard(summary, lift, stability) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
+    const metrics = aggregateMetrics(results.scenarios);
+    if (metrics.total_reps > 0) {
+        const tokens = metrics.input_tokens === null
+            ? "subject tokens unavailable"
+            : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
+        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
+        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported) · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
+    }
     // Lift is a statement about a skill-delivered run (green or force). On a red run
     // the caller may still have a lift in hand (a scored run exists in the same tag),
     // but printing it under a baseline scorecard reads as if the baseline itself

package/dist/sandbox.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Optional OS-containment seam.
+ *
+ * The current workspace implementation copies fixtures into temp directories;
+ * that is isolation for test data, NOT process/filesystem/network containment.
+ * No production backend is registered yet. A container/bwrap implementation can
+ * satisfy this interface without teaching scoring or trajectory gates about one
+ * particular sandbox runtime.
+ */
+export type SandboxNetworkPolicy = "allow" | "deny";
+export interface SandboxRequest {
+    sourceWorkspace: string;
+    network: SandboxNetworkPolicy;
+}
+export interface SandboxSession {
+    cwd: string;
+    captureDiff(): Promise<string>;
+    cleanup(): Promise<void>;
+}
+export interface SandboxBackend {
+    name: string;
+    /** `os` is a backend claim; `none` must never be presented as containment. */
+    containment: "os" | "none";
+    prepare(request: SandboxRequest): Promise<SandboxSession>;
+}
+export interface SandboxRunResult<T> {
+    value: T;
+    diff: string;
+    backend: string;
+    containment: SandboxBackend["containment"];
+}
+export declare function withSandbox<T>(backend: SandboxBackend, request: SandboxRequest, run: (session: SandboxSession) => Promise<T>): Promise<SandboxRunResult<T>>;

package/dist/sandbox.js

@@ -0,0 +1,12 @@
+export async function withSandbox(backend, request, run) {
+    const session = await backend.prepare(request);
+    try {
+        const value = await run(session);
+        const diff = await session.captureDiff();
+        return { value, diff, backend: backend.name, containment: backend.containment };
+    }
+    finally {
+        await session.cleanup();
+    }
+}
+//# sourceMappingURL=sandbox.js.map

package/dist/score.d.ts

@@ -22,12 +22,14 @@ export interface ScoreResult {
     criticalFails: number;
     bSeriesFails: number;
     suspectCount: number;
+    errorCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL and ERROR both count against it. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
+ * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total

package/dist/score.js

@@ -11,7 +11,8 @@ export function letterFor(pct) {
 }
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL and ERROR both count against it. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
+ * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total
@@ -25,11 +26,16 @@ export function score(verdicts, input) {
     let criticalFails = 0;
     let bSeriesFails = 0;
     let suspectCount = 0;
+    let errorCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
             continue; // untrustworthy: neither pass nor fail
         }
+        if (v.verdict === "ERROR" || v.verdict === "JUDGE-AMBIGUOUS") {
+            errorCount++;
+            continue; // infrastructure/indeterminate: neither behavioral pass nor fail
+        }
         total++;
         if (v.verdict === "PASS") {
             passed++;
@@ -42,21 +48,30 @@ export function score(verdicts, input) {
     }
     const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
     const letter = letterFor(pct);
-    const ship = total >= shipBar.total &&
+    const validBar = Number.isInteger(shipBar.total) && shipBar.total >= 1 && Number.isInteger(shipBar.min_pass) && shipBar.min_pass >= 1 && shipBar.min_pass <= shipBar.total;
+    const ship = validBar &&
+        total >= shipBar.total &&
         passed >= shipBar.min_pass &&
         (!shipBar.no_critical_fail || criticalFails === 0) &&
         bSeriesFails === 0 &&
-        suspectCount === 0;
+        suspectCount === 0 &&
+        errorCount === 0;
     let note = "";
-    if (suspectCount > 0) {
+    if (!validBar) {
+        note = "invalid ship bar: total/min_pass must be positive integers with min_pass <= total";
+    }
+    else if (suspectCount > 0) {
         note = `${suspectCount} suspect: re-judge/resolve`;
     }
+    else if (errorCount > 0) {
+        note = `${errorCount} infrastructure error${errorCount === 1 ? "" : "s"}: retry/repair evidence`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/seeded.d.ts

@@ -1,5 +1,6 @@
 import type { Scenario } from "./spec.js";
 import type { ExecutionTraceV1 } from "./capture-trace-types.js";
+import type { TrajectoryEventV1 } from "./trajectory-gates.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
@@ -42,9 +43,14 @@ export type VitestRun = ExecResult;
 export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
+    gateError: string | null;
     diff: string;
-    /** One per turn; empty unless the scenario declared `assert.trace`. */
+    /** One per turn; empty unless structured execution was requested. */
     traces: ExecutionTraceV1[];
+    /** Adapter-neutral workflow/tool events for assert.trajectory. */
+    events: TrajectoryEventV1[];
+    /** Required native sources that could not be read or normalized. */
+    eventErrors: string[];
 }
 /**
  * The added/removed lines of a unified diff — what the model actually *changed*,