@skill-harness/core 0.7.0 → 0.9.0

package/dist/restamp.js

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import yaml from "js-yaml";
+import { loadSpec } from "./spec.js";
+import { resultsPath } from "./results.js";
+import { enumerateRunDirs } from "./lint.js";
+import { promptDocDigest, SKILL_KEY, SKILL_PROMPT_KEY, PROMPT_PREFIX, UNREADABLE, } from "./sources.js";
+/**
+ * The `source_hashes` map of a raw-loaded results doc, or null when the doc has none.
+ * Read from the RAW yaml rather than through `readResults` on purpose: this rewrites the
+ * file, and `readResults` migrates schema 1 → 2 in memory, so writing its output back
+ * would silently upgrade the schema of records nobody asked to touch.
+ */
+function rawHashes(doc) {
+    if (!doc || typeof doc !== "object")
+        return null;
+    const h = doc.source_hashes;
+    if (!h || typeof h !== "object" || Array.isArray(h))
+        return null;
+    return h;
+}
+/**
+ * `git show <ref>:<path>` as BYTES, or null when the ref, the repo, or the path isn't there.
+ *
+ * `execFileSync` with a buffer encoding rather than the shared `exec` helper: that one
+ * accumulates stdout as decoded strings, so a multi-byte character landing on a chunk
+ * boundary would come back with a replacement character and hash to something the run
+ * never recorded. This comparison is byte-exact by definition, so it reads bytes.
+ */
+function bytesAtRef(abs, ref) {
+    const cwd = dirname(abs);
+    try {
+        const top = execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd, encoding: "utf8" }).trim();
+        const rel = relative(top, abs).split("\\").join("/");
+        return execFileSync("git", ["show", `${ref}:${rel}`], { cwd, maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return null; // no git, no repo, no such ref, or the file did not exist there
+    }
+}
+/**
+ * The bytes a run is claimed to have measured: `<ref>`'s copy, or the working tree's when
+ * no ref was given. Null when they cannot be read at all, which proves nothing and so
+ * upgrades nothing.
+ */
+function measuredBytes(abs, from) {
+    if (from !== undefined)
+        return bytesAtRef(abs, from);
+    try {
+        return readFileSync(abs);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fail loudly when `--from` names something git cannot resolve.
+ *
+ * Without this the whole migration degrades into "0 upgraded, everything left alone",
+ * which is indistinguishable from a corpus that genuinely cannot be proven — measured on
+ * a clone whose only local branch was the feature branch, where `--from main` silently
+ * did nothing and read as the tool being broken. A ref the operator got wrong is their
+ * mistake to see, not one to absorb.
+ */
+function assertRefResolves(dir, ref) {
+    try {
+        execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd: dir, stdio: "ignore" });
+    }
+    catch {
+        throw new Error(`--from \`${ref}\` does not resolve to a commit in the repo holding ${dir} — ` +
+            `check the ref exists locally (a fresh clone often has no local \`main\`; try \`origin/main\`)`);
+    }
+}
+/**
+ * Write via a temp file in the same directory, then rename.
+ *
+ * `rename(2)` is atomic within a filesystem, so a reader sees the old file or the new one
+ * and never a half-written one. It matters more here than in the writers that produce a
+ * run: this loops over an entire published corpus rewriting records that already hold
+ * measurements nobody can reproduce without spending model tokens, so an interrupt or a
+ * full disk partway through would truncate exactly the artifacts the command exists to
+ * preserve. The temp file is `.tmp`-suffixed beside the target rather than in the system
+ * temp dir, because a cross-device rename is not atomic — and is not even permitted.
+ */
+function writeAtomically(path, text) {
+    const tmp = `${path}.restamp.tmp`;
+    writeFileSync(tmp, text, "utf8");
+    try {
+        renameSync(tmp, path);
+    }
+    catch (e) {
+        try {
+            rmSync(tmp, { force: true });
+        }
+        catch { /* the rename failure is what matters */ }
+        throw e;
+    }
+}
+/** Upgrade every provable record under one skill's tests/results. */
+export function restampSkill(skillDir, opts = {}) {
+    const report = { runs: 0, upgraded: 0, unprovable: 0, unchanged: 0, partial: 0, added: [] };
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    const spec = loadSpec(specPath);
+    const specDir = dirname(specPath);
+    // Only `system_prompt_file` paths, resolved from the CURRENT spec. The other
+    // path-shaped keys (`post_test`, `env.extensions`) name code, not prompt documents:
+    // every byte of those reaches the run, so they keep hashing raw and are left alone.
+    const promptDocs = new Map([[SKILL_KEY, resolve(skillDir, "SKILL.md")]]);
+    for (const s of spec.scenarios) {
+        if (s.systemPromptFile)
+            promptDocs.set(s.systemPromptFile, resolve(specDir, s.systemPromptFile));
+    }
+    if (opts.from !== undefined)
+        assertRefResolves(skillDir, opts.from);
+    // Resolved once per document, not once per run: with `--from`, each resolution is two
+    // git invocations, and a corpus of 140 runs would otherwise pay for them 140 times over.
+    const provable = new Map();
+    for (const [legacyKey, abs] of promptDocs) {
+        const measured = measuredBytes(abs, opts.from);
+        let currentPrompt = null;
+        try {
+            currentPrompt = promptDocDigest(readFileSync(abs, "utf8"));
+        }
+        catch {
+            currentPrompt = null;
+        }
+        provable.set(legacyKey, {
+            measuredRaw: measured === null ? null : createHash("sha256").update(measured).digest("hex"),
+            measuredPrompt: measured === null ? null : promptDocDigest(measured.toString("utf8")),
+            currentPrompt,
+        });
+    }
+    for (const runDir of enumerateRunDirs(join(skillDir, "tests", "results"))) {
+        report.runs++;
+        let doc;
+        try {
+            doc = yaml.load(readFileSync(resultsPath(runDir), "utf8"));
+        }
+        catch {
+            // Unreadable/malformed → lint already reports it; never rewrite what we can't read.
+            // Still bucketed: every record counted in `runs` has to be accounted for somewhere.
+            report.unchanged++;
+            continue;
+        }
+        const hashes = rawHashes(doc);
+        if (!hashes) {
+            report.unchanged++; // predates source_hashes → nothing to upgrade from
+            continue;
+        }
+        let blocked = false;
+        const additions = [];
+        for (const legacyKey of promptDocs.keys()) {
+            const recorded = hashes[legacyKey];
+            if (recorded === undefined || recorded === UNREADABLE)
+                continue; // never measured → nothing proven
+            const upgradedKey = legacyKey === SKILL_KEY ? SKILL_PROMPT_KEY : PROMPT_PREFIX + legacyKey;
+            if (hashes[upgradedKey] !== undefined)
+                continue; // already upgraded
+            const p = provable.get(legacyKey);
+            if (p.currentPrompt === null)
+                continue; // the file is gone; lint reports that itself
+            // Two steps, and both are required. First: this run measured the reference bytes —
+            // otherwise the reference says nothing about THIS run. Second: those bytes and
+            // today's carry the same model-visible text — otherwise what the model reads has
+            // genuinely moved and only a re-run can speak for it.
+            if (p.measuredRaw === null || recorded !== p.measuredRaw || p.measuredPrompt !== p.currentPrompt) {
+                blocked = true;
+                continue;
+            }
+            additions.push([upgradedKey, p.currentPrompt]);
+        }
+        if (additions.length === 0) {
+            // Exclusive: a record that gained nothing is `unprovable` only if something
+            // actually blocked it, and `unchanged` otherwise.
+            if (blocked)
+                report.unprovable++;
+            else
+                report.unchanged++;
+            continue;
+        }
+        for (const [k, v] of additions) {
+            hashes[k] = v;
+            report.added.push(`${runDir}: ${k}`);
+        }
+        writeAtomically(resultsPath(runDir), yaml.dump(doc, { lineWidth: 100 }));
+        report.upgraded++;
+        if (blocked)
+            report.partial++;
+    }
+    return report;
+}
+//# sourceMappingURL=restamp.js.map

package/dist/results.d.ts

@@ -2,6 +2,23 @@ import { type ModelRef } from "./adapters/types.js";
 import { type ScenarioVerdict } from "./score.js";
 import type { Verdict } from "./score.js";
 import type { ShipBar, Scenario } from "./spec.js";
+export interface ScenarioMetrics {
+    wall_time_ms: number;
+    judge_calls: number;
+    judge_rejudge_calls: number;
+    subject_metrics_reps: number;
+    total_reps: number;
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_read_tokens?: number;
+    cache_write_tokens?: number;
+    subject_cost_usd?: number;
+    tool_calls?: number;
+    delegated_children?: number;
+    max_concurrency?: number;
+}
+/** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
+export declare function mergeScenarioMetrics(prior: ScenarioMetrics | undefined, fresh: ScenarioMetrics | undefined): ScenarioMetrics | undefined;
 export interface ScenarioResult {
     id: string;
     judge_verdict: Verdict;
@@ -14,6 +31,8 @@ export interface ScenarioResult {
     clean?: number;
     flakiness?: number;
     pass_threshold?: number;
+    /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
+    metrics?: ScenarioMetrics;
     /**
      * Objective trace-gate evidence. ADDITIVE and optional.
      *
@@ -59,6 +78,11 @@ export interface ObjectiveResult {
     status: "PASS" | "FAIL" | "ERROR";
     trace_version?: number;
     trace_sha256?: string;
+    trajectory_version?: string;
+    events_sha256?: string;
+    /** Per-repetition hashes retained when an aggregate has more than one rep. */
+    rep_events_sha256?: string[];
+    rep_trace_sha256?: string[];
     assertions: {
         kind: string;
         status: "PASS" | "FAIL" | "ERROR";
@@ -327,6 +351,10 @@ export declare function rebuildScenarioResult(fresh: ScenarioResult, prior: Scen
  * traces append without a wrapper.
  */
 export declare function tracePath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** Saved adapter-neutral workflow events for one scenario rep. */
+export declare function trajectoryPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's normalized-event files, sorted (plain first, then numeric rep). */
+export declare function findTrajectoryFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */

package/dist/results.js

@@ -4,6 +4,29 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+/** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
+export function mergeScenarioMetrics(prior, fresh) {
+    if (!prior)
+        return fresh;
+    if (!fresh)
+        return prior;
+    const subject = prior.subject_metrics_reps > 0 ? prior : fresh;
+    return {
+        wall_time_ms: prior.wall_time_ms + fresh.wall_time_ms,
+        judge_calls: prior.judge_calls + fresh.judge_calls,
+        judge_rejudge_calls: prior.judge_rejudge_calls + fresh.judge_rejudge_calls,
+        subject_metrics_reps: subject.subject_metrics_reps,
+        total_reps: prior.total_reps,
+        ...(subject.input_tokens === undefined ? {} : { input_tokens: subject.input_tokens }),
+        ...(subject.output_tokens === undefined ? {} : { output_tokens: subject.output_tokens }),
+        ...(subject.cache_read_tokens === undefined ? {} : { cache_read_tokens: subject.cache_read_tokens }),
+        ...(subject.cache_write_tokens === undefined ? {} : { cache_write_tokens: subject.cache_write_tokens }),
+        ...(subject.subject_cost_usd === undefined ? {} : { subject_cost_usd: subject.subject_cost_usd }),
+        ...(subject.tool_calls === undefined ? {} : { tool_calls: subject.tool_calls }),
+        ...(subject.delegated_children === undefined ? {} : { delegated_children: subject.delegated_children }),
+        ...(subject.max_concurrency === undefined ? {} : { max_concurrency: subject.max_concurrency }),
+    };
+}
 /**
  * The run modes whose results carry a real grade: the ones where the skill under
  * test was actually delivered to the model.
@@ -53,6 +76,11 @@ export function scoreContextFor(run, spec) {
 }
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export function effectiveThreshold(prevScenario, scenario) {
+    // Critical includes destructive/security/capability/data-loss AND right-sizing
+    // counterexamples. Every clean repetition must pass; an aggregate majority may
+    // not hide one release-blocking failure.
+    if (scenario.critical)
+        return 1;
     return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
 }
 /** Slugify an ISO timestamp into a filesystem-safe directory name. */
@@ -252,7 +280,7 @@ export function ensureResultsGitignore(resultsRoot) {
 // so `repIndexOf` returned null for traces and `regate` looked for an unsuffixed
 // path that does not exist on a multi-rep run — reporting "trace missing" for
 // traces sitting on disk.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?(?:txt|trace\.jsonl)$/;
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:(?:judge|diff)\.txt|(?:trace|events)\.jsonl|txt)$/;
 /** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
@@ -356,7 +384,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -367,7 +395,14 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         return p === "fresh" ? freshValue : priorValue;
     };
     const objective = pick(policy.objective, freshObjective, prior?.objective);
-    const adjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" ||
+        (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+    // A settled cell-level panel cannot overrule newly evaluated objective gates,
+    // nor can one transcript erase a failed repetition under an all-clean policy.
+    const adjudication = conflictsWithFreshEvidence && pickedAdjudication
+        ? { ...pickedAdjudication, state: "unresolved", verdict: undefined }
+        : pickedAdjudication;
     // `unresolved` is carried by the `suspect` flag and by nothing else — the
     // adjudication block records WHY, but `suspect` is what the ship bar reads.
     // Taking `suspect` from `fresh` while carrying the block therefore published a
@@ -382,7 +417,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     // `<id>.judge.txt` would revert a `confirmed`/`tie_broken` verdict that two or
     // three judges settled — leaving `adjudication.verdict` on the record
     // contradicting the `judge_verdict` beside it.
-    const settled = policy.adjudication === "carry" ? adjudication?.verdict : undefined;
+    const settled = policy.adjudication === "carry" && !conflictsWithFreshEvidence ? adjudication?.verdict : undefined;
     // Field ORDER matches `outcomesToResult`, the writer that produces a run's
     // results.yaml in the first place. It is not cosmetic: `results.yaml` is a
     // committed file, and emitting the same fields in a different order made every
@@ -400,6 +435,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(clean === undefined ? {} : { clean }),
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
         // The author owns the verdict; a re-measurement never discards their call.
         override: prior?.override ?? null,
         note: prior?.note ?? "",
@@ -421,6 +457,21 @@ export function tracePath(runDir, scenarioId, mode, rep) {
     const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
     return join(runDir, `${base}.trace.jsonl`);
 }
+/** Saved adapter-neutral workflow events for one scenario rep. */
+export function trajectoryPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.events.jsonl`);
+}
+/** A scenario's normalized-event files, sorted (plain first, then numeric rep). */
+export function findTrajectoryFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.events\\.jsonl$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.events\\.jsonl$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export function findDiffFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -469,6 +520,7 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         // override whose justification was gitignored, and left `regate` with nothing
         // to re-evaluate on the one cell a human had disputed.
         ...findTraceFiles(runDir, scenarioId),
+        ...findTrajectoryFiles(runDir, scenarioId),
     ];
     if (files.length === 0)
         return;

package/dist/run.js

@@ -2,18 +2,20 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
 import { evaluateTraceGates } from "./trace-gates.js";
+import { evaluateTrajectoryGates, serializeTrajectoryEvents, } from "./trajectory-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
+import { aggregateMetrics } from "./comparison.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -107,7 +109,9 @@ export async function runSkillModel(opts) {
     const grouped = scenarios.map(() => []);
     flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
     const scenarioResults = scenarios.map((scenario, si) => {
-        const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
+        const threshold = scenario.critical
+            ? effectiveThreshold(undefined, scenario)
+            : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
@@ -152,6 +156,7 @@ export function hasEmptyAssistantTurn(transcript) {
 }
 /** Run ONE rep of a scenario in its own isolated workspace. */
 async function runRep(scenario, rep, repCount, ctx) {
+    const startedAt = performance.now();
     const { spec, judge, mode, runDir, now, log } = ctx;
     const repField = repCount > 1 ? { rep } : {};
     if (rep === 0) {
@@ -161,6 +166,7 @@ async function runRep(scenario, rep, repCount, ctx) {
     let ws = null;
     let transcript = "";
     let gatePrefix = null;
+    let infrastructureFailure = null;
     // Null until a seeded rep actually reaches its gates: a workspace-setup failure
     // produces no diff, and writing an empty artifact there would misreport "the
     // model changed nothing" for a rep that never ran.
@@ -170,12 +176,16 @@ async function runRep(scenario, rep, repCount, ctx) {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
         }
         catch (e) {
-            // A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
+            // A setup failure means the scenario never executed. It blocks release as
+            // infrastructure ERROR; it is not evidence of subject behavior.
             gatePrefix = e instanceof Error ? e.message : String(e);
+            infrastructureFailure = gatePrefix;
             transcript = `[workspace setup failed] ${gatePrefix}`;
         }
         let noResponse = false;
         let traces = [];
+        let events = [];
+        let eventErrors = [];
         let unobservablePaths = false;
         // The pre-run state, captured AFTER `createWorkspace` has applied the
         // fixture's `_staged/` and `_uncommitted/` trees. Those land after the
@@ -201,12 +211,15 @@ async function runRep(scenario, rep, repCount, ctx) {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
                         specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
-                        trace: scenario.traceAssert ? { scenarioId: scenario.id, rep } : undefined,
+                        trace: scenario.traceAssert || scenario.trajectoryAssert ? { scenarioId: scenario.id, rep } : undefined,
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
+                    infrastructureFailure = r.gateError;
                     stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
                     traces = r.traces;
+                    events = r.events;
+                    eventErrors = r.eventErrors;
                 }
                 else {
                     const req = {
@@ -217,18 +230,21 @@ async function runRep(scenario, rep, repCount, ctx) {
                             : undefined,
                         // Absolute before it reaches a child process running in a neutral cwd.
                         extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
+                        eventSources: scenario.eventSources,
                     };
-                    if (scenario.traceAssert) {
+                    if (scenario.traceAssert || scenario.trajectoryAssert) {
                         // Missing required evidence is ERROR, never a silent fallback to the
                         // unstructured path: a gate with nothing to read must not look like a
                         // gate that passed.
                         if (!ctx.adapter.runStructured) {
-                            throw new Error(`scenario \`${scenario.id}\` declares \`assert.trace\`, but the \`${ctx.adapter.name}\` adapter` +
-                                ` cannot produce execution traces — the gate would have no evidence to read.`);
+                            throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
+                                ` cannot produce execution traces/events — the gate would have no evidence to read.`);
                         }
                         const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
                         transcript = structured.transcript;
                         traces = structured.traces;
+                        events = structured.events ?? [];
+                        eventErrors = structured.eventErrors ?? [];
                     }
                     else {
                         transcript = await ctx.adapter.run(req);
@@ -279,34 +295,55 @@ async function runRep(scenario, rep, repCount, ctx) {
         // Objective evidence is persisted for every rep, pass or fail — a failing gate
         // is exactly when someone wants to read what the model actually did.
         let objective;
-        if (scenario.traceAssert) {
-            if (traces.length > 0) {
-                writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
-            }
-            const merged = mergeTraces(traces);
-            if (unobservablePaths) {
-                gatePrefix = "objective: workspace changes could not be observed — `unchanged_paths` has no evidence to check";
-                objective = { status: "ERROR", assertions: [] };
-            }
-            else if (merged === null) {
-                // Declared a gate, produced no trace: that is broken infrastructure, and
-                // grading it either way would be inventing a result.
-                gatePrefix = "objective: no execution trace was produced — cannot evaluate assert.trace";
-                objective = { status: "ERROR", assertions: [] };
+        if (scenario.traceAssert || scenario.trajectoryAssert) {
+            const assertionResults = [];
+            let status = "PASS";
+            let traceMeta = {};
+            let trajectoryMeta = {};
+            if (scenario.traceAssert) {
+                if (traces.length > 0) {
+                    writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+                }
+                const merged = mergeTraces(traces);
+                if (unobservablePaths) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "unchanged_path", status: "ERROR", detail: "workspace changes could not be observed" });
+                }
+                else if (merged === null) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "trace_evidence", status: "ERROR", detail: "no execution trace was produced" });
+                }
+                else {
+                    const gate = evaluateTraceGates(scenario.traceAssert, merged);
+                    status = gate.status;
+                    assertionResults.push(...gate.assertions);
+                    traceMeta = { trace_version: merged.trace_version, trace_sha256: merged.trace_sha256 };
+                }
             }
-            else {
-                const gate = evaluateTraceGates(scenario.traceAssert, merged);
-                objective = {
-                    status: gate.status,
-                    trace_version: merged.trace_version,
-                    trace_sha256: merged.trace_sha256,
-                    assertions: gate.assertions,
-                };
-                if (gate.status === "FAIL") {
-                    // Set the same gatePrefix the seeded gates use, so a trace failure
-                    // short-circuits the judge through the path that already exists.
-                    gatePrefix = `objective: ${gate.assertions.filter((x) => x.status === "FAIL").map((x) => x.detail).join("; ")}`;
+            if (scenario.trajectoryAssert) {
+                if (events.length > 0) {
+                    writeFileSync(trajectoryPath(runDir, scenario.id, mode, repSuffix), serializeTrajectoryEvents(events), "utf8");
+                }
+                if (eventErrors.length > 0) {
+                    status = "ERROR";
+                    assertionResults.push(...eventErrors.map((detail) => ({ kind: "trajectory_evidence", status: "ERROR", detail })));
                 }
+                else if (events.length === 0) {
+                    status = "ERROR";
+                    assertionResults.push({ kind: "trajectory_evidence", status: "ERROR", detail: "no normalized workflow events were produced" });
+                }
+                else {
+                    const gate = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
+                    if (gate.status === "ERROR" || (gate.status === "FAIL" && status === "PASS"))
+                        status = gate.status;
+                    assertionResults.push(...gate.assertions);
+                    trajectoryMeta = { trajectory_version: gate.event_version, events_sha256: gate.events_sha256 };
+                }
+            }
+            objective = { status, ...traceMeta, ...trajectoryMeta, assertions: assertionResults };
+            if (status !== "PASS") {
+                const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
+                gatePrefix = `objective: ${details.join("; ") || "structured evidence could not be evaluated"}`;
             }
             appendJournal(runDir, {
                 event: "objective-result", ts: now(), id: scenario.id,
@@ -316,11 +353,17 @@ async function runRep(scenario, rep, repCount, ctx) {
         let verdict;
         let reason;
         let suspect = false;
+        let judgeCalls = 0;
         if (objective?.status === "ERROR") {
             verdict = "ERROR";
             reason = gatePrefix ?? "objective evidence missing";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
         }
+        else if (infrastructureFailure) {
+            verdict = "ERROR";
+            reason = infrastructureFailure;
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
         else if (noResponse) {
             verdict = "ERROR";
             reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
@@ -340,9 +383,19 @@ async function runRep(scenario, rep, repCount, ctx) {
             verdict = o.verdict;
             reason = o.reason;
             suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
+            judgeCalls = 1;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        return { verdict, reason, suspect, objective };
+        const subject = mergeTraces(traces)?.metrics;
+        return {
+            verdict, reason, suspect, objective,
+            metrics: {
+                wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
+                judge_calls: judgeCalls,
+                judge_rejudge_calls: 0,
+                ...(subject ? { subject } : {}),
+            },
+        };
     }
     finally {
         ws?.cleanup();
@@ -370,7 +423,7 @@ export function formatScorecard(summary, lift, stability) {
     const lines = [];
     lines.push(`── ${results.skill} · ${results.harness} · ${results.model} ──`);
     for (const s of results.scenarios) {
-        const v = s.override ?? s.judge_verdict;
+        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
         const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : "?";
         const ov = s.override ? " (override)" : "";
         const susp = s.suspect ? " ⚠suspect" : "";
@@ -381,6 +434,14 @@ export function formatScorecard(summary, lift, stability) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
+    const metrics = aggregateMetrics(results.scenarios);
+    if (metrics.total_reps > 0) {
+        const tokens = metrics.input_tokens === null
+            ? "subject tokens unavailable"
+            : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
+        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
+        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported) · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
+    }
     // Lift is a statement about a skill-delivered run (green or force). On a red run
     // the caller may still have a lift in hand (a scored run exists in the same tag),
     // but printing it under a baseline scorecard reads as if the baseline itself

package/dist/sandbox.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Optional OS-containment seam.
+ *
+ * The current workspace implementation copies fixtures into temp directories;
+ * that is isolation for test data, NOT process/filesystem/network containment.
+ * No production backend is registered yet. A container/bwrap implementation can
+ * satisfy this interface without teaching scoring or trajectory gates about one
+ * particular sandbox runtime.
+ */
+export type SandboxNetworkPolicy = "allow" | "deny";
+export interface SandboxRequest {
+    sourceWorkspace: string;
+    network: SandboxNetworkPolicy;
+}
+export interface SandboxSession {
+    cwd: string;
+    captureDiff(): Promise<string>;
+    cleanup(): Promise<void>;
+}
+export interface SandboxBackend {
+    name: string;
+    /** `os` is a backend claim; `none` must never be presented as containment. */
+    containment: "os" | "none";
+    prepare(request: SandboxRequest): Promise<SandboxSession>;
+}
+export interface SandboxRunResult<T> {
+    value: T;
+    diff: string;
+    backend: string;
+    containment: SandboxBackend["containment"];
+}
+export declare function withSandbox<T>(backend: SandboxBackend, request: SandboxRequest, run: (session: SandboxSession) => Promise<T>): Promise<SandboxRunResult<T>>;