@skill-harness/core 0.5.0 → 0.7.0

package/dist/results.d.ts

@@ -14,6 +14,56 @@ export interface ScenarioResult {
     clean?: number;
     flakiness?: number;
     pass_threshold?: number;
+    /**
+     * Objective trace-gate evidence. ADDITIVE and optional.
+     *
+     * Absent means "the scenario declared no trace assertions" — NOT an objective
+     * pass. Anything reading this must treat the two as different; collapsing them
+     * would silently upgrade every legacy result to "objectively verified".
+     */
+    objective?: ObjectiveResult;
+    /**
+     * Confidence-aware adjudication. ADDITIVE and optional.
+     *
+     * Absent means historical single-judge behavior — NOT that judges agreed. An
+     * unresolved adjudication additionally sets `suspect: true`, which is what
+     * actually blocks SHIP; this field is the audit trail behind that flag.
+     */
+    adjudication?: AdjudicationResult;
+}
+/** One judge's answer for a cell, kept verbatim however the collapse turned out. */
+export interface Judgment {
+    /** 1-based: judgment 1 is the first-wave judge, 2 the secondary, 3 the tie-break. */
+    ordinal: number;
+    judge: {
+        provider: string;
+        model: string;
+    };
+    verdict: Verdict;
+    reason: string;
+    /** The judge misfired — recorded, never counted as a clean vote. */
+    suspect: boolean;
+}
+export interface AdjudicationResult {
+    state: "confirmed" | "tie_broken" | "unresolved";
+    /** Why the cell was re-judged. */
+    trigger: "ambiguous" | "contradictory" | "non_unanimous" | "ship_deciding";
+    /** Every judgment, in order. Never pruned — an author resolving this needs all of them. */
+    judgments: Judgment[];
+    /** The collapsed answer. Absent when unresolved. */
+    verdict?: "PASS" | "FAIL";
+}
+/** Objective gate outcome for one scenario cell. */
+export interface ObjectiveResult {
+    /** ERROR means the evidence was missing or malformed — never a pass. */
+    status: "PASS" | "FAIL" | "ERROR";
+    trace_version?: number;
+    trace_sha256?: string;
+    assertions: {
+        kind: string;
+        status: "PASS" | "FAIL" | "ERROR";
+        detail: string;
+    }[];
 }
 export interface GradeSummary {
     passed: number;
@@ -56,16 +106,23 @@ export interface ResultsFile {
     harness_cli_version?: string;
     /**
      * `pass` when this run proved, before spending the wave, that the skill body was
-     * reachable in the model's context (see canary.ts). Absent means the probe was
-     * not asked for — never that it failed, because a failed canary aborts the run
-     * and no results.yaml is written.
+     * reachable in the model's context (see canary.ts). `skipped` when the probe was
+     * asked for but could not be performed — SKILL.md has no `## ` heading to quote
+     * back, so no reply could prove anything. Absent means the probe was not asked
+     * for — never that it failed, because a failed canary aborts the run and no
+     * results.yaml is written.
+     *
+     * `skipped` exists because absent and skipped were the same value: a user who
+     * passed `--canary` precisely because pi ≥ 0.83.0 delivery is unreliable got a
+     * silently degraded probe, a fully billed wave, and a committed results.yaml
+     * byte-identical to a run where delivery was never checked at all.
      *
      * Only green runs can carry it: red delivers nothing by design and force delivers
      * through the system prompt. It is provenance for the *validity* of a green run,
      * which is why it lives here rather than only in the journal — `journal.jsonl` is
      * gitignored, and this claim has to survive a commit.
      */
-    delivery_canary?: "pass";
+    delivery_canary?: "pass" | "skipped";
     skill: string;
     harness: string;
     model: string;
@@ -152,12 +209,35 @@ export declare function runDirFor(skillDir: string, harness: string, model: Mode
 export declare function transcriptPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 export declare function reportPath(runDir: string): string;
 export declare function resultsPath(runDir: string): string;
-/** The verdict that counts: author override when present, else the judge's. */
+/**
+ * The verdict that counts: author override when present, else the objective
+ * gate, else the judge's.
+ *
+ * **The objective gate outranks the judge.** `assert.trace` is a mechanical
+ * statement about what the model DID — it called `write`, it touched `.env`.
+ * The judge is an LLM reading prose. When they disagree, the measurement wins.
+ *
+ * This is the only place that ordering is enforced, and it has to be here.
+ * `objective` used to reach the ship decision solely through `gatePrefix` in
+ * `run.ts`, which forces a single rep's verdict — so every path that recomputed
+ * a verdict afterwards silently dropped the gate while keeping the `objective`
+ * block that claimed it was enforced. Three of them did: `--reps N` out-voted an
+ * objective FAIL 2-to-1 (100%, grade A, SHIP, on a CRITICAL scenario that called
+ * a forbidden tool), `regrade` re-judged from a transcript the tool calls are
+ * absent from, and `regate` recomputed it. `reps.ts` already states the policy —
+ * "one rep that called a forbidden tool is a real finding, not a minority draw
+ * to be voted away" — and nothing enforced it.
+ *
+ * An author override still wins, exactly as it does over `suspect`. Overriding a
+ * deterministic assertion is a deliberate, recorded human act — and the failure
+ * this guards against was never a human deciding, it was nobody deciding.
+ */
 export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,
  * so a persisted grade can never disagree with verdicts + overrides.
- * ctx is null for unscored (red/force) runs.
+ * ctx is null for unscored runs — `red` only, since 0.5.0: `force` is a real
+ * deployment and is scored (see SCORED_MODES directly above).
  */
 export declare function finalizeResults(draft: ResultsDraft, ctx: ScoreContext | null): ResultsFile;
 /** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
@@ -207,8 +287,50 @@ export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mo
  * generated evidence, ignored like transcripts, not committed like results.yaml.
  */
 export declare function diffPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/**
+ * What happens to one piece of recorded evidence when a command rebuilds a result.
+ *
+ *  - `carry` — the command did not re-measure this, so the prior value still
+ *    describes the run and must survive.
+ *  - `fresh` — the command re-measured it; take the new value.
+ *  - `drop`  — the command invalidated it; a stale value would misinform.
+ */
+export type EvidencePolicy = "carry" | "fresh" | "drop";
+export interface RebuildPolicy {
+    objective: EvidencePolicy;
+    adjudication: EvidencePolicy;
+}
+/**
+ * Rebuild a `ScenarioResult` after a command re-measured part of it.
+ *
+ * **The single choke point for every rewriter**, and exhaustive by construction:
+ * every field is destructured below, so adding one to `ScenarioResult` fails the
+ * build HERE until someone decides whether it is carried, taken fresh, or
+ * dropped. That guard is the whole point of the function.
+ *
+ * It exists because the ad-hoc version — `{ ...fresh, override: prior.override,
+ * note: prior.note }`, written independently in three places — silently dropped
+ * `objective` from `grade` and `adjudication` from `regate`. Both failures ran in
+ * the dangerous direction: a trace-gated scenario re-read as "no assertions
+ * declared", and an unresolved judge disagreement as a settled verdict. 1,036
+ * tests passed through it; a real smoke run caught it.
+ *
+ * The author's `override` and `note` are always carried and are not policy —
+ * no command re-measures a human's judgement.
+ */
+export declare function rebuildScenarioResult(fresh: ScenarioResult, prior: ScenarioResult | undefined, policy: RebuildPolicy): ScenarioResult;
+/**
+ * Where a rep's execution trace is saved: `<id>.<mode>[.rep<k>].trace.jsonl`.
+ *
+ * `.jsonl` rather than `.txt` so it is distinguishable at a glance from a
+ * transcript, and one JSON object per line so a multi-turn scenario's per-turn
+ * traces append without a wrapper.
+ */
+export declare function tracePath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findTraceFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
 /**

package/dist/results.js

@@ -74,18 +74,58 @@ export function reportPath(runDir) {
 export function resultsPath(runDir) {
     return join(runDir, "results.yaml");
 }
-/** The verdict that counts: author override when present, else the judge's. */
+/**
+ * The verdict that counts: author override when present, else the objective
+ * gate, else the judge's.
+ *
+ * **The objective gate outranks the judge.** `assert.trace` is a mechanical
+ * statement about what the model DID — it called `write`, it touched `.env`.
+ * The judge is an LLM reading prose. When they disagree, the measurement wins.
+ *
+ * This is the only place that ordering is enforced, and it has to be here.
+ * `objective` used to reach the ship decision solely through `gatePrefix` in
+ * `run.ts`, which forces a single rep's verdict — so every path that recomputed
+ * a verdict afterwards silently dropped the gate while keeping the `objective`
+ * block that claimed it was enforced. Three of them did: `--reps N` out-voted an
+ * objective FAIL 2-to-1 (100%, grade A, SHIP, on a CRITICAL scenario that called
+ * a forbidden tool), `regrade` re-judged from a transcript the tool calls are
+ * absent from, and `regate` recomputed it. `reps.ts` already states the policy —
+ * "one rep that called a forbidden tool is a real finding, not a minority draw
+ * to be voted away" — and nothing enforced it.
+ *
+ * An author override still wins, exactly as it does over `suspect`. Overriding a
+ * deterministic assertion is a deliberate, recorded human act — and the failure
+ * this guards against was never a human deciding, it was nobody deciding.
+ */
 export function effectiveVerdicts(scenarios) {
     return scenarios.map((s) => ({
         id: s.id,
-        verdict: s.override ?? s.judge_verdict,
+        verdict: s.override ?? objectiveVerdict(s) ?? s.judge_verdict,
         suspect: s.suspect && s.override == null, // an override resolves the misfire
     }));
 }
+/**
+ * The verdict an objective gate forces, or undefined when it forces nothing.
+ *
+ * ERROR outranks FAIL: "the evidence is missing" must never read as "the
+ * assertion held". Absent `objective` forces nothing at all — it means the
+ * scenario declared no trace assertions, and treating that as a pass would
+ * upgrade every legacy result to "objectively verified".
+ */
+function objectiveVerdict(s) {
+    if (!s.objective)
+        return undefined;
+    if (s.objective.status === "ERROR")
+        return "ERROR";
+    if (s.objective.status === "FAIL")
+        return "FAIL";
+    return undefined;
+}
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,
  * so a persisted grade can never disagree with verdicts + overrides.
- * ctx is null for unscored (red/force) runs.
+ * ctx is null for unscored runs — `red` only, since 0.5.0: `force` is a real
+ * deployment and is scored (see SCORED_MODES directly above).
  */
 export function finalizeResults(draft, ctx) {
     let effective_grade;
@@ -208,7 +248,11 @@ export function ensureResultsGitignore(resultsRoot) {
 }
 // Matches transcript (`.rep<k>.txt`), judge-raw (`.rep<k>.judge.txt`) and
 // staged-diff (`.rep<k>.diff.txt`) rep suffixes.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?txt$/;
+// Every rep-suffixed artifact kind. `.trace.jsonl` was added without updating this,
+// so `repIndexOf` returned null for traces and `regate` looked for an unsuffixed
+// path that does not exist on a multi-rep run — reporting "trace missing" for
+// traces sitting on disk.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?(?:txt|trace\.jsonl)$/;
 /** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
@@ -268,9 +312,13 @@ export function findJudgeRawFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
         return [];
     const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // `judge2`/`judge3` are the second and third opinions adjudication writes. They
+    // were not matched here, so an override on an adjudicated cell committed the
+    // first judge's answer and silently dropped the very judgments the adjudication
+    // rested on — the audit trail minus its evidence.
     const re = mode === undefined
-        ? new RegExp(`^${esc}\\..*\\.judge\\.txt$`)
-        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
+        ? new RegExp(`^${esc}\\..*\\.judge\\d*\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\d*\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
 /**
@@ -287,6 +335,92 @@ export function diffPath(runDir, scenarioId, mode, rep) {
     const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
     return join(runDir, `${base}.diff.txt`);
 }
+/**
+ * Rebuild a `ScenarioResult` after a command re-measured part of it.
+ *
+ * **The single choke point for every rewriter**, and exhaustive by construction:
+ * every field is destructured below, so adding one to `ScenarioResult` fails the
+ * build HERE until someone decides whether it is carried, taken fresh, or
+ * dropped. That guard is the whole point of the function.
+ *
+ * It exists because the ad-hoc version — `{ ...fresh, override: prior.override,
+ * note: prior.note }`, written independently in three places — silently dropped
+ * `objective` from `grade` and `adjudication` from `regate`. Both failures ran in
+ * the dangerous direction: a trace-gated scenario re-read as "no assertions
+ * declared", and an unresolved judge disagreement as a settled verdict. 1,036
+ * tests passed through it; a real smoke run caught it.
+ *
+ * The author's `override` and `note` are always carried and are not policy —
+ * no command re-measures a human's judgement.
+ */
+export function rebuildScenarioResult(fresh, prior, policy) {
+    // Exhaustive destructure. Do not replace with a spread: the spread is what
+    // allowed a new field to pass through unconsidered in the first place.
+    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const _exhaustive = rest;
+    void _exhaustive;
+    void _freshOverride;
+    void _freshNote;
+    const pick = (p, freshValue, priorValue) => {
+        if (p === "drop")
+            return undefined;
+        return p === "fresh" ? freshValue : priorValue;
+    };
+    const objective = pick(policy.objective, freshObjective, prior?.objective);
+    const adjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    // `unresolved` is carried by the `suspect` flag and by nothing else — the
+    // adjudication block records WHY, but `suspect` is what the ship bar reads.
+    // Taking `suspect` from `fresh` while carrying the block therefore published a
+    // record that said `state: "unresolved"` and scored as a clean SHIP. `regate`
+    // did exactly that: it rebuilds verdict and suspect from the saved first-wave
+    // judge file, so a free, offline command silently resolved a disagreement in
+    // favour of shipping — inverting this module's stated invariant that an
+    // unresolved disagreement must not resolve itself.
+    const unresolved = adjudication?.state === "unresolved";
+    // A settled adjudication outranks a re-read of the first-wave judge file for
+    // the same reason: `regate` re-measures GATES, not judgments, so re-reading
+    // `<id>.judge.txt` would revert a `confirmed`/`tie_broken` verdict that two or
+    // three judges settled — leaving `adjudication.verdict` on the record
+    // contradicting the `judge_verdict` beside it.
+    const settled = policy.adjudication === "carry" ? adjudication?.verdict : undefined;
+    // Field ORDER matches `outcomesToResult`, the writer that produces a run's
+    // results.yaml in the first place. It is not cosmetic: `results.yaml` is a
+    // committed file, and emitting the same fields in a different order made every
+    // `grade`/`regate`/`adjudicate` rewrite every multi-rep scenario block in the
+    // corpus with a pure-noise diff.
+    return {
+        id,
+        judge_verdict: settled ?? judge_verdict,
+        judge_reason,
+        suspect: suspect || unresolved,
+        // Aggregation shape always comes from the fresh computation — these describe
+        // how THIS result was aggregated, not the previous one.
+        ...(reps === undefined ? {} : { reps }),
+        ...(passes === undefined ? {} : { passes }),
+        ...(clean === undefined ? {} : { clean }),
+        ...(flakiness === undefined ? {} : { flakiness }),
+        ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        // The author owns the verdict; a re-measurement never discards their call.
+        override: prior?.override ?? null,
+        note: prior?.note ?? "",
+        // Omitted rather than set to undefined: absent must stay absent, so a result
+        // with no evidence serialises byte-identically to one from before the field
+        // existed.
+        ...(objective ? { objective } : {}),
+        ...(adjudication ? { adjudication } : {}),
+    };
+}
+/**
+ * Where a rep's execution trace is saved: `<id>.<mode>[.rep<k>].trace.jsonl`.
+ *
+ * `.jsonl` rather than `.txt` so it is distinguishable at a glance from a
+ * transcript, and one JSON object per line so a multi-turn scenario's per-turn
+ * traces append without a wrapper.
+ */
+export function tracePath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.trace.jsonl`);
+}
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export function findDiffFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -297,6 +431,16 @@ export function findDiffFiles(runDir, scenarioId, mode) {
         : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.diff\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
+/** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findTraceFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.trace\\.jsonl$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.trace\\.jsonl$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export function findTranscriptFile(runDir, scenarioId) {
     return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
@@ -320,6 +464,11 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         ...findTranscriptFiles(runDir, scenarioId),
         ...findJudgeRawFiles(runDir, scenarioId),
         ...findDiffFiles(runDir, scenarioId),
+        // On a trace-gated scenario the trace IS the evidence for the override — the
+        // same role the staged diff plays on a seeded one. Omitting it committed an
+        // override whose justification was gitignored, and left `regate` with nothing
+        // to re-evaluate on the one cell a human had disputed.
+        ...findTraceFiles(runDir, scenarioId),
     ];
     if (files.length === 0)
         return;

package/dist/run.d.ts

@@ -2,6 +2,7 @@ import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
+import { type ScenarioStability } from "./stability.js";
 export interface RunOptions {
     spec: Spec;
     skillDir: string;
@@ -55,5 +56,12 @@ export declare function hasEmptyAssistantTurn(transcript: string): boolean;
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;
+export declare function formatScorecard(summary: RunSummary, lift?: Lift, stability?: ScenarioStability[]): string;

package/dist/run.js

@@ -2,15 +2,18 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
-import { createWorkspace } from "./workspace.js";
+import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { evaluateTraceGates } from "./trace-gates.js";
+import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
+import { boundaryCells, stabilityNote } from "./stability.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -76,8 +79,13 @@ export async function runSkillModel(opts) {
         });
         if (canary.status === "fail")
             throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
-        if (canary.status === "skipped")
+        if (canary.status === "skipped") {
+            // Recorded, not just logged: `journal.jsonl` is gitignored, and the claim
+            // "this run's delivery was verified" has to survive a commit — including
+            // when it is the claim that it wasn't.
+            canaryStatus = "skipped";
             log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        }
         else {
             canaryStatus = "pass";
             log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
@@ -167,6 +175,14 @@ async function runRep(scenario, rep, repCount, ctx) {
             transcript = `[workspace setup failed] ${gatePrefix}`;
         }
         let noResponse = false;
+        let traces = [];
+        let unobservablePaths = false;
+        // The pre-run state, captured AFTER `createWorkspace` has applied the
+        // fixture's `_staged/` and `_uncommitted/` trees. Those land after the
+        // baseline commit, so a fixture that ships a deliberately dirty tree was
+        // being reported as changes the model made — a fabricated FAIL, written into
+        // a committed results.yaml, naming files the model never touched.
+        let before = ws ? snapshotPaths(ws.cwd, scenario.workspace) : null;
         if (ws) {
             // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
             // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
@@ -177,24 +193,46 @@ async function runRep(scenario, rep, repCount, ctx) {
                     log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
                     ws.cleanup();
                     ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
+                    // A fresh workspace needs a fresh baseline, or the retry's diff would
+                    // be taken against a directory that no longer exists.
+                    before = snapshotPaths(ws.cwd, scenario.workspace);
                 }
                 if (scenario.mode === "seeded") {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
                         specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
+                        trace: scenario.traceAssert ? { scenarioId: scenario.id, rep } : undefined,
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
                     stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
+                    traces = r.traces;
                 }
                 else {
-                    transcript = await ctx.adapter.run({
+                    const req = {
                         skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
                         // resolved like fixtures: relative to the spec's dir
                         systemPromptFile: scenario.systemPromptFile
                             ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
                             : undefined,
-                    });
+                        // Absolute before it reaches a child process running in a neutral cwd.
+                        extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
+                    };
+                    if (scenario.traceAssert) {
+                        // Missing required evidence is ERROR, never a silent fallback to the
+                        // unstructured path: a gate with nothing to read must not look like a
+                        // gate that passed.
+                        if (!ctx.adapter.runStructured) {
+                            throw new Error(`scenario \`${scenario.id}\` declares \`assert.trace\`, but the \`${ctx.adapter.name}\` adapter` +
+                                ` cannot produce execution traces — the gate would have no evidence to read.`);
+                        }
+                        const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
+                        transcript = structured.transcript;
+                        traces = structured.traces;
+                    }
+                    else {
+                        transcript = await ctx.adapter.run(req);
+                    }
                 }
                 noResponse = hasEmptyAssistantTurn(transcript);
                 if (!noResponse)
@@ -213,10 +251,77 @@ async function runRep(scenario, rep, repCount, ctx) {
             }
             appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
         }
+        // Filesystem evidence for `unchanged_paths`, observed AFTER the model ran.
+        //
+        // It cannot come through `RunReq`: the request is built before the run, and
+        // what changed only exists afterwards. That plumbing existed and no caller
+        // ever set it, so `changed_paths` was always `[]` and every
+        // `unchanged_paths` assertion passed vacuously — a safety gate reporting
+        // green while the model rewrote the workspace.
+        // Only when the scenario actually asserts on paths. A scenario using only
+        // `require_calls` / `forbid_calls` needs no filesystem evidence, so a
+        // workspace it cannot observe is not an error for it.
+        if (scenario.traceAssert?.unchanged_paths?.length && traces.length > 0 && ws) {
+            const changed = diffSnapshots(before, snapshotPaths(ws.cwd, scenario.workspace));
+            if (changed === null) {
+                // Missing evidence is ERROR, never a pass — spec.ts refuses the
+                // `workspace: none` combination up front, so reaching here means the
+                // workspace could not be read.
+                unobservablePaths = true;
+            }
+            else {
+                traces = traces.map((t) => {
+                    const withPaths = { ...t, changed_paths: changed };
+                    return { ...withPaths, trace_sha256: traceSha256(withPaths) };
+                });
+            }
+        }
+        // Objective evidence is persisted for every rep, pass or fail — a failing gate
+        // is exactly when someone wants to read what the model actually did.
+        let objective;
+        if (scenario.traceAssert) {
+            if (traces.length > 0) {
+                writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+            }
+            const merged = mergeTraces(traces);
+            if (unobservablePaths) {
+                gatePrefix = "objective: workspace changes could not be observed — `unchanged_paths` has no evidence to check";
+                objective = { status: "ERROR", assertions: [] };
+            }
+            else if (merged === null) {
+                // Declared a gate, produced no trace: that is broken infrastructure, and
+                // grading it either way would be inventing a result.
+                gatePrefix = "objective: no execution trace was produced — cannot evaluate assert.trace";
+                objective = { status: "ERROR", assertions: [] };
+            }
+            else {
+                const gate = evaluateTraceGates(scenario.traceAssert, merged);
+                objective = {
+                    status: gate.status,
+                    trace_version: merged.trace_version,
+                    trace_sha256: merged.trace_sha256,
+                    assertions: gate.assertions,
+                };
+                if (gate.status === "FAIL") {
+                    // Set the same gatePrefix the seeded gates use, so a trace failure
+                    // short-circuits the judge through the path that already exists.
+                    gatePrefix = `objective: ${gate.assertions.filter((x) => x.status === "FAIL").map((x) => x.detail).join("; ")}`;
+                }
+            }
+            appendJournal(runDir, {
+                event: "objective-result", ts: now(), id: scenario.id,
+                ok: objective.status === "PASS", detail: gatePrefix ?? "", ...repField,
+            });
+        }
         let verdict;
         let reason;
         let suspect = false;
-        if (noResponse) {
+        if (objective?.status === "ERROR") {
+            verdict = "ERROR";
+            reason = gatePrefix ?? "objective evidence missing";
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
+        else if (noResponse) {
             verdict = "ERROR";
             reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
@@ -237,7 +342,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        return { verdict, reason, suspect };
+        return { verdict, reason, suspect, objective };
     }
     finally {
         ws?.cleanup();
@@ -251,8 +356,15 @@ async function runRep(scenario, rep, repCount, ctx) {
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export function formatScorecard(summary, lift) {
+export function formatScorecard(summary, lift, stability) {
     const { results } = summary;
     const g = results.effective_grade;
     const lines = [];
@@ -283,12 +395,20 @@ export function formatScorecard(summary, lift) {
     // Said on the scorecard, not just in the docs: the one thing that can invalidate
     // a green number is invisible in the number. `harness_cli_version` is recorded
     // beside the verdicts so a reader can tell which pi produced them.
-    if (results.mode === "green" && !results.delivery_canary) {
+    // `skipped` counts as unproven here, not as proven: the probe was asked for and
+    // could not answer, which leaves delivery exactly as unverified as never asking.
+    if (results.mode === "green" && results.delivery_canary !== "pass") {
         lines.push(`  NOTE:  green delivery is harness-version-dependent` +
             (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
             ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
             ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
     }
+    // Boundary cells last, because they qualify the verdicts above: a ✓ on a cell that
+    // flipped between the last two runs is one draw, whatever its rep count said.
+    const ran = new Set(results.scenarios.map((s) => s.id));
+    for (const s of boundaryCells(stability ?? []).filter((c) => ran.has(c.id))) {
+        lines.push(`  ⇄ ${stabilityNote(s)}`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/seeded.d.ts

@@ -1,4 +1,5 @@
 import type { Scenario } from "./spec.js";
+import type { ExecutionTraceV1 } from "./capture-trace-types.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
@@ -17,6 +18,14 @@ interface SeededOpts {
      * deterministically.
      */
     runVitest?: (args: string[], cwd: string) => Promise<VitestRun>;
+    /**
+     * Trace metadata. Present when the scenario declares `assert.trace`, which
+     * routes the subject through the adapter's structured (`--mode json`) path.
+     */
+    trace?: {
+        scenarioId: string;
+        rep: number;
+    };
 }
 /**
  * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
@@ -34,6 +43,8 @@ export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
     diff: string;
+    /** One per turn; empty unless the scenario declared `assert.trace`. */
+    traces: ExecutionTraceV1[];
 }
 /**
  * The added/removed lines of a unified diff — what the model actually *changed*,