@skill-harness/core 0.6.0 → 0.7.0

package/dist/results.js

@@ -74,18 +74,58 @@ export function reportPath(runDir) {
 export function resultsPath(runDir) {
     return join(runDir, "results.yaml");
 }
-/** The verdict that counts: author override when present, else the judge's. */
+/**
+ * The verdict that counts: author override when present, else the objective
+ * gate, else the judge's.
+ *
+ * **The objective gate outranks the judge.** `assert.trace` is a mechanical
+ * statement about what the model DID — it called `write`, it touched `.env`.
+ * The judge is an LLM reading prose. When they disagree, the measurement wins.
+ *
+ * This is the only place that ordering is enforced, and it has to be here.
+ * `objective` used to reach the ship decision solely through `gatePrefix` in
+ * `run.ts`, which forces a single rep's verdict — so every path that recomputed
+ * a verdict afterwards silently dropped the gate while keeping the `objective`
+ * block that claimed it was enforced. Three of them did: `--reps N` out-voted an
+ * objective FAIL 2-to-1 (100%, grade A, SHIP, on a CRITICAL scenario that called
+ * a forbidden tool), `regrade` re-judged from a transcript the tool calls are
+ * absent from, and `regate` recomputed it. `reps.ts` already states the policy —
+ * "one rep that called a forbidden tool is a real finding, not a minority draw
+ * to be voted away" — and nothing enforced it.
+ *
+ * An author override still wins, exactly as it does over `suspect`. Overriding a
+ * deterministic assertion is a deliberate, recorded human act — and the failure
+ * this guards against was never a human deciding, it was nobody deciding.
+ */
 export function effectiveVerdicts(scenarios) {
     return scenarios.map((s) => ({
         id: s.id,
-        verdict: s.override ?? s.judge_verdict,
+        verdict: s.override ?? objectiveVerdict(s) ?? s.judge_verdict,
         suspect: s.suspect && s.override == null, // an override resolves the misfire
     }));
 }
+/**
+ * The verdict an objective gate forces, or undefined when it forces nothing.
+ *
+ * ERROR outranks FAIL: "the evidence is missing" must never read as "the
+ * assertion held". Absent `objective` forces nothing at all — it means the
+ * scenario declared no trace assertions, and treating that as a pass would
+ * upgrade every legacy result to "objectively verified".
+ */
+function objectiveVerdict(s) {
+    if (!s.objective)
+        return undefined;
+    if (s.objective.status === "ERROR")
+        return "ERROR";
+    if (s.objective.status === "FAIL")
+        return "FAIL";
+    return undefined;
+}
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,
  * so a persisted grade can never disagree with verdicts + overrides.
- * ctx is null for unscored (red/force) runs.
+ * ctx is null for unscored runs — `red` only, since 0.5.0: `force` is a real
+ * deployment and is scored (see SCORED_MODES directly above).
  */
 export function finalizeResults(draft, ctx) {
     let effective_grade;
@@ -208,7 +248,11 @@ export function ensureResultsGitignore(resultsRoot) {
 }
 // Matches transcript (`.rep<k>.txt`), judge-raw (`.rep<k>.judge.txt`) and
 // staged-diff (`.rep<k>.diff.txt`) rep suffixes.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?txt$/;
+// Every rep-suffixed artifact kind. `.trace.jsonl` was added without updating this,
+// so `repIndexOf` returned null for traces and `regate` looked for an unsuffixed
+// path that does not exist on a multi-rep run — reporting "trace missing" for
+// traces sitting on disk.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?(?:txt|trace\.jsonl)$/;
 /** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
@@ -268,9 +312,13 @@ export function findJudgeRawFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
         return [];
     const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // `judge2`/`judge3` are the second and third opinions adjudication writes. They
+    // were not matched here, so an override on an adjudicated cell committed the
+    // first judge's answer and silently dropped the very judgments the adjudication
+    // rested on — the audit trail minus its evidence.
     const re = mode === undefined
-        ? new RegExp(`^${esc}\\..*\\.judge\\.txt$`)
-        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
+        ? new RegExp(`^${esc}\\..*\\.judge\\d*\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\d*\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
 /**
@@ -287,6 +335,92 @@ export function diffPath(runDir, scenarioId, mode, rep) {
     const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
     return join(runDir, `${base}.diff.txt`);
 }
+/**
+ * Rebuild a `ScenarioResult` after a command re-measured part of it.
+ *
+ * **The single choke point for every rewriter**, and exhaustive by construction:
+ * every field is destructured below, so adding one to `ScenarioResult` fails the
+ * build HERE until someone decides whether it is carried, taken fresh, or
+ * dropped. That guard is the whole point of the function.
+ *
+ * It exists because the ad-hoc version — `{ ...fresh, override: prior.override,
+ * note: prior.note }`, written independently in three places — silently dropped
+ * `objective` from `grade` and `adjudication` from `regate`. Both failures ran in
+ * the dangerous direction: a trace-gated scenario re-read as "no assertions
+ * declared", and an unresolved judge disagreement as a settled verdict. 1,036
+ * tests passed through it; a real smoke run caught it.
+ *
+ * The author's `override` and `note` are always carried and are not policy —
+ * no command re-measures a human's judgement.
+ */
+export function rebuildScenarioResult(fresh, prior, policy) {
+    // Exhaustive destructure. Do not replace with a spread: the spread is what
+    // allowed a new field to pass through unconsidered in the first place.
+    const { id, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, objective: freshObjective, adjudication: freshAdjudication, ...rest } = fresh;
+    const _exhaustive = rest;
+    void _exhaustive;
+    void _freshOverride;
+    void _freshNote;
+    const pick = (p, freshValue, priorValue) => {
+        if (p === "drop")
+            return undefined;
+        return p === "fresh" ? freshValue : priorValue;
+    };
+    const objective = pick(policy.objective, freshObjective, prior?.objective);
+    const adjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
+    // `unresolved` is carried by the `suspect` flag and by nothing else — the
+    // adjudication block records WHY, but `suspect` is what the ship bar reads.
+    // Taking `suspect` from `fresh` while carrying the block therefore published a
+    // record that said `state: "unresolved"` and scored as a clean SHIP. `regate`
+    // did exactly that: it rebuilds verdict and suspect from the saved first-wave
+    // judge file, so a free, offline command silently resolved a disagreement in
+    // favour of shipping — inverting this module's stated invariant that an
+    // unresolved disagreement must not resolve itself.
+    const unresolved = adjudication?.state === "unresolved";
+    // A settled adjudication outranks a re-read of the first-wave judge file for
+    // the same reason: `regate` re-measures GATES, not judgments, so re-reading
+    // `<id>.judge.txt` would revert a `confirmed`/`tie_broken` verdict that two or
+    // three judges settled — leaving `adjudication.verdict` on the record
+    // contradicting the `judge_verdict` beside it.
+    const settled = policy.adjudication === "carry" ? adjudication?.verdict : undefined;
+    // Field ORDER matches `outcomesToResult`, the writer that produces a run's
+    // results.yaml in the first place. It is not cosmetic: `results.yaml` is a
+    // committed file, and emitting the same fields in a different order made every
+    // `grade`/`regate`/`adjudicate` rewrite every multi-rep scenario block in the
+    // corpus with a pure-noise diff.
+    return {
+        id,
+        judge_verdict: settled ?? judge_verdict,
+        judge_reason,
+        suspect: suspect || unresolved,
+        // Aggregation shape always comes from the fresh computation — these describe
+        // how THIS result was aggregated, not the previous one.
+        ...(reps === undefined ? {} : { reps }),
+        ...(passes === undefined ? {} : { passes }),
+        ...(clean === undefined ? {} : { clean }),
+        ...(flakiness === undefined ? {} : { flakiness }),
+        ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        // The author owns the verdict; a re-measurement never discards their call.
+        override: prior?.override ?? null,
+        note: prior?.note ?? "",
+        // Omitted rather than set to undefined: absent must stay absent, so a result
+        // with no evidence serialises byte-identically to one from before the field
+        // existed.
+        ...(objective ? { objective } : {}),
+        ...(adjudication ? { adjudication } : {}),
+    };
+}
+/**
+ * Where a rep's execution trace is saved: `<id>.<mode>[.rep<k>].trace.jsonl`.
+ *
+ * `.jsonl` rather than `.txt` so it is distinguishable at a glance from a
+ * transcript, and one JSON object per line so a multi-turn scenario's per-turn
+ * traces append without a wrapper.
+ */
+export function tracePath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.trace.jsonl`);
+}
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export function findDiffFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -297,6 +431,16 @@ export function findDiffFiles(runDir, scenarioId, mode) {
         : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.diff\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
+/** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findTraceFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.trace\\.jsonl$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.trace\\.jsonl$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export function findTranscriptFile(runDir, scenarioId) {
     return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
@@ -320,6 +464,11 @@ export function preserveTranscript(resultsRoot, runDir, scenarioId) {
         ...findTranscriptFiles(runDir, scenarioId),
         ...findJudgeRawFiles(runDir, scenarioId),
         ...findDiffFiles(runDir, scenarioId),
+        // On a trace-gated scenario the trace IS the evidence for the override — the
+        // same role the staged diff plays on a seeded one. Omitting it committed an
+        // override whose justification was gitignored, and left `regate` with nothing
+        // to re-evaluate on the one cell a human had disputed.
+        ...findTraceFiles(runDir, scenarioId),
     ];
     if (files.length === 0)
         return;

package/dist/run.js

@@ -2,11 +2,13 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
-import { createWorkspace } from "./workspace.js";
+import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { evaluateTraceGates } from "./trace-gates.js";
+import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
@@ -77,8 +79,13 @@ export async function runSkillModel(opts) {
         });
         if (canary.status === "fail")
             throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
-        if (canary.status === "skipped")
+        if (canary.status === "skipped") {
+            // Recorded, not just logged: `journal.jsonl` is gitignored, and the claim
+            // "this run's delivery was verified" has to survive a commit — including
+            // when it is the claim that it wasn't.
+            canaryStatus = "skipped";
             log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        }
         else {
             canaryStatus = "pass";
             log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
@@ -168,6 +175,14 @@ async function runRep(scenario, rep, repCount, ctx) {
             transcript = `[workspace setup failed] ${gatePrefix}`;
         }
         let noResponse = false;
+        let traces = [];
+        let unobservablePaths = false;
+        // The pre-run state, captured AFTER `createWorkspace` has applied the
+        // fixture's `_staged/` and `_uncommitted/` trees. Those land after the
+        // baseline commit, so a fixture that ships a deliberately dirty tree was
+        // being reported as changes the model made — a fabricated FAIL, written into
+        // a committed results.yaml, naming files the model never touched.
+        let before = ws ? snapshotPaths(ws.cwd, scenario.workspace) : null;
         if (ws) {
             // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
             // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
@@ -178,24 +193,46 @@ async function runRep(scenario, rep, repCount, ctx) {
                     log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
                     ws.cleanup();
                     ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
+                    // A fresh workspace needs a fresh baseline, or the retry's diff would
+                    // be taken against a directory that no longer exists.
+                    before = snapshotPaths(ws.cwd, scenario.workspace);
                 }
                 if (scenario.mode === "seeded") {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
                         specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
+                        trace: scenario.traceAssert ? { scenarioId: scenario.id, rep } : undefined,
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
                     stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
+                    traces = r.traces;
                 }
                 else {
-                    transcript = await ctx.adapter.run({
+                    const req = {
                         skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
                         // resolved like fixtures: relative to the spec's dir
                         systemPromptFile: scenario.systemPromptFile
                             ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
                             : undefined,
-                    });
+                        // Absolute before it reaches a child process running in a neutral cwd.
+                        extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
+                    };
+                    if (scenario.traceAssert) {
+                        // Missing required evidence is ERROR, never a silent fallback to the
+                        // unstructured path: a gate with nothing to read must not look like a
+                        // gate that passed.
+                        if (!ctx.adapter.runStructured) {
+                            throw new Error(`scenario \`${scenario.id}\` declares \`assert.trace\`, but the \`${ctx.adapter.name}\` adapter` +
+                                ` cannot produce execution traces — the gate would have no evidence to read.`);
+                        }
+                        const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
+                        transcript = structured.transcript;
+                        traces = structured.traces;
+                    }
+                    else {
+                        transcript = await ctx.adapter.run(req);
+                    }
                 }
                 noResponse = hasEmptyAssistantTurn(transcript);
                 if (!noResponse)
@@ -214,10 +251,77 @@ async function runRep(scenario, rep, repCount, ctx) {
             }
             appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
         }
+        // Filesystem evidence for `unchanged_paths`, observed AFTER the model ran.
+        //
+        // It cannot come through `RunReq`: the request is built before the run, and
+        // what changed only exists afterwards. That plumbing existed and no caller
+        // ever set it, so `changed_paths` was always `[]` and every
+        // `unchanged_paths` assertion passed vacuously — a safety gate reporting
+        // green while the model rewrote the workspace.
+        // Only when the scenario actually asserts on paths. A scenario using only
+        // `require_calls` / `forbid_calls` needs no filesystem evidence, so a
+        // workspace it cannot observe is not an error for it.
+        if (scenario.traceAssert?.unchanged_paths?.length && traces.length > 0 && ws) {
+            const changed = diffSnapshots(before, snapshotPaths(ws.cwd, scenario.workspace));
+            if (changed === null) {
+                // Missing evidence is ERROR, never a pass — spec.ts refuses the
+                // `workspace: none` combination up front, so reaching here means the
+                // workspace could not be read.
+                unobservablePaths = true;
+            }
+            else {
+                traces = traces.map((t) => {
+                    const withPaths = { ...t, changed_paths: changed };
+                    return { ...withPaths, trace_sha256: traceSha256(withPaths) };
+                });
+            }
+        }
+        // Objective evidence is persisted for every rep, pass or fail — a failing gate
+        // is exactly when someone wants to read what the model actually did.
+        let objective;
+        if (scenario.traceAssert) {
+            if (traces.length > 0) {
+                writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
+            }
+            const merged = mergeTraces(traces);
+            if (unobservablePaths) {
+                gatePrefix = "objective: workspace changes could not be observed — `unchanged_paths` has no evidence to check";
+                objective = { status: "ERROR", assertions: [] };
+            }
+            else if (merged === null) {
+                // Declared a gate, produced no trace: that is broken infrastructure, and
+                // grading it either way would be inventing a result.
+                gatePrefix = "objective: no execution trace was produced — cannot evaluate assert.trace";
+                objective = { status: "ERROR", assertions: [] };
+            }
+            else {
+                const gate = evaluateTraceGates(scenario.traceAssert, merged);
+                objective = {
+                    status: gate.status,
+                    trace_version: merged.trace_version,
+                    trace_sha256: merged.trace_sha256,
+                    assertions: gate.assertions,
+                };
+                if (gate.status === "FAIL") {
+                    // Set the same gatePrefix the seeded gates use, so a trace failure
+                    // short-circuits the judge through the path that already exists.
+                    gatePrefix = `objective: ${gate.assertions.filter((x) => x.status === "FAIL").map((x) => x.detail).join("; ")}`;
+                }
+            }
+            appendJournal(runDir, {
+                event: "objective-result", ts: now(), id: scenario.id,
+                ok: objective.status === "PASS", detail: gatePrefix ?? "", ...repField,
+            });
+        }
         let verdict;
         let reason;
         let suspect = false;
-        if (noResponse) {
+        if (objective?.status === "ERROR") {
+            verdict = "ERROR";
+            reason = gatePrefix ?? "objective evidence missing";
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
+        else if (noResponse) {
             verdict = "ERROR";
             reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
@@ -238,7 +342,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
-        return { verdict, reason, suspect };
+        return { verdict, reason, suspect, objective };
     }
     finally {
         ws?.cleanup();
@@ -291,7 +395,9 @@ export function formatScorecard(summary, lift, stability) {
     // Said on the scorecard, not just in the docs: the one thing that can invalidate
     // a green number is invisible in the number. `harness_cli_version` is recorded
     // beside the verdicts so a reader can tell which pi produced them.
-    if (results.mode === "green" && !results.delivery_canary) {
+    // `skipped` counts as unproven here, not as proven: the probe was asked for and
+    // could not answer, which leaves delivery exactly as unverified as never asking.
+    if (results.mode === "green" && results.delivery_canary !== "pass") {
         lines.push(`  NOTE:  green delivery is harness-version-dependent` +
             (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
             ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +

package/dist/seeded.d.ts

@@ -1,4 +1,5 @@
 import type { Scenario } from "./spec.js";
+import type { ExecutionTraceV1 } from "./capture-trace-types.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
@@ -17,6 +18,14 @@ interface SeededOpts {
      * deterministically.
      */
     runVitest?: (args: string[], cwd: string) => Promise<VitestRun>;
+    /**
+     * Trace metadata. Present when the scenario declares `assert.trace`, which
+     * routes the subject through the adapter's structured (`--mode json`) path.
+     */
+    trace?: {
+        scenarioId: string;
+        rep: number;
+    };
 }
 /**
  * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
@@ -34,6 +43,8 @@ export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
     diff: string;
+    /** One per turn; empty unless the scenario declared `assert.trace`. */
+    traces: ExecutionTraceV1[];
 }
 /**
  * The added/removed lines of a unified diff — what the model actually *changed*,

package/dist/seeded.js

@@ -160,13 +160,37 @@ export function capDiff(diff, maxBytes = DIFF_MAX_BYTES) {
  */
 export async function runSeeded(scenario, opts) {
     const repo = opts.cwd;
-    const harnessOut = await opts.adapter.run({
+    const req = {
         skillDir: opts.skillDir,
         model: opts.model,
         mode: opts.mode,
         turns: scenario.turns,
         cwd: repo,
-    });
+        // Resolved against the spec dir, exactly like fixtures and post-tests.
+        extensions: scenario.extensions?.map((e) => resolve(opts.specDir, e)),
+    };
+    // A trace-gated seeded scenario runs through the structured path so the tool
+    // calls are recorded; everything downstream (gates, diff, transcript) is
+    // identical, because the rebuilt transcript is what print mode would have
+    // emitted anyway.
+    let traces = [];
+    let harnessOut;
+    if (opts.trace) {
+        if (!opts.adapter.runStructured) {
+            throw new Error(`scenario \`${opts.trace.scenarioId}\` declares \`assert.trace\`, but the \`${opts.adapter.name}\` adapter` +
+                ` cannot produce execution traces — the gate would have no evidence to read.`);
+        }
+        const structured = await opts.adapter.runStructured({
+            ...req,
+            scenarioId: opts.trace.scenarioId,
+            rep: opts.trace.rep,
+        });
+        harnessOut = structured.transcript;
+        traces = structured.traces;
+    }
+    else {
+        harnessOut = await opts.adapter.run(req);
+    }
     const parts = [harnessOut, "", "=== SEEDED GATES ==="];
     let gateFailure = null;
     const runVitest = opts.runVitest ??
@@ -189,7 +213,7 @@ export async function runSeeded(scenario, opts) {
             (gitFailure.stderr.trim() ? `: ${gitFailure.stderr.trim().split("\n")[0]}` : "");
         parts.push(`  staged diff: ERROR (${msg})`);
         gateFailure = msg;
-        return finish(parts, gateFailure, diff);
+        return finish(parts, gateFailure, diff, traces);
     }
     // BOTH needle gates read the changed lines only, never context. A unified diff
     // carries three lines of context per hunk, so an untouched symbol near the edit
@@ -257,7 +281,7 @@ export async function runSeeded(scenario, opts) {
                 parts.push(`  post_test: ERROR (${msg})`);
                 if (!gateFailure)
                     gateFailure = msg;
-                return finish(parts, gateFailure, diff);
+                return finish(parts, gateFailure, diff, traces);
             }
             const v = await runVitest([POST_TEST_BASE], repo);
             const out = `${v.stdout}\n${v.stderr}`;
@@ -306,7 +330,7 @@ export async function runSeeded(scenario, opts) {
                 gateFailure = problem;
         }
     }
-    return finish(parts, gateFailure, diff);
+    return finish(parts, gateFailure, diff, traces);
 }
 function git(cwd, args) {
     return exec("git", args, { cwd, timeoutMs: 30_000 });
@@ -340,13 +364,13 @@ function bothStreams(v) {
     return o || e;
 }
 /** Append the staged diff and return the outcome. Every exit path goes through here, so the judge always sees the same sections in the same order. */
-function finish(parts, gateFailure, diff) {
+function finish(parts, gateFailure, diff, traces = []) {
     // The code itself, last — the gates above only prove that keywords appeared.
     // Without this section a seeded checklist item about what the code *does* is
     // graded from the model's own description of its work.
     parts.push("", "=== STAGED DIFF ===");
     parts.push(diff.trim() === "" ? "  (empty — the model left no staged changes)" : capDiff(diff));
-    return { transcript: parts.join("\n"), gateFailure, diff };
+    return { transcript: parts.join("\n"), gateFailure, diff, traces };
 }
 /**
  * Parse vitest's `Tests  N passed | M skipped (T)` summary line.

package/dist/sources.js

@@ -167,24 +167,50 @@ function walk(dir, prefix = "") {
  * transcripts are actually invalid.
  */
 function facets(s) {
-    const { id, title, critical, mode, turns, checklist, fixture, assert, workspace, remote, systemPromptFile, reps, passThreshold, ...restScenario } = s;
+    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
+    // `covers` is destructured into a discard on purpose, and this comment is the
+    // decision the guard demanded: it belongs to NO digest. It records which
+    // instruction sections a scenario is declared to exercise, which changes what
+    // `--affected` selects next time — not what any past run measured. Bucketing it
+    // anywhere would charge a re-run (or at best a re-score) for editing a label,
+    // which is the exact trap the facet split was built to remove.
+    void _coversIsMetadata;
     const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
     const _assertExhaustive = restAssert;
     void _assertExhaustive;
-    const hasGates = diff_contains !== undefined || diff_excludes !== undefined;
+    // `traceAssert` is a GATE, not stimulus: it is evaluated against a trace the run
+    // already saved, so `regate` (free) can re-answer it without re-running the model.
+    // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
+    // changes what gets executed, the other only what we conclude from it.
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
     return {
         // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
         // run executes in the workspace, and neither can be re-evaluated from a saved
         // diff. (`post_test`'s CONTENTS get their own file-path key, hashed separately.)
+        // `extensions` is STIMULUS, not a gate — note the asymmetry with `traceAssert`
+        // below. Changing which extensions load changes what the model can DO, so the
+        // old transcripts describe a different agent and only a re-run can answer.
+        // Changing an assertion only changes what we conclude from evidence already on
+        // disk, which `regate` can redo for free.
+        // APPENDED CONDITIONALLY, never as a fixed slot. This tuple is positional and
+        // its hash is stored in every published results.yaml, so adding an
+        // unconditional element re-hashes every scenario that never used the field —
+        // measured: 62 real lint findings became 261 across the reference corpus, all of
+        // them demanding paid re-runs for scenarios nobody had edited.
         stimulus: JSON.stringify([
             id, mode, turns, workspace, remote, systemPromptFile ?? null,
             fixture ?? null, vitest ?? null, post_test ?? null,
+            ...(extensions ? [extensions] : []),
         ]),
         rubric: JSON.stringify([id, title, checklist]),
         policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
-        gates: hasGates ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null]) : null,
+        // Same rule as `stimulus` above: conditional, so a needle-gated scenario that
+        // declares no trace assertions keeps the digest it was published with.
+        gates: hasGates
+            ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null, ...(traceAssert ? [traceAssert] : [])])
+            : null,
     };
 }
 function sha(canonical) {
@@ -281,6 +307,15 @@ export function sourceHashes(ctx) {
         if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
             hashes[s.systemPromptFile] = fileSha256(resolve(ctx.specDir, s.systemPromptFile)) ?? UNREADABLE;
         }
+        // Extension CONTENTS, not just the paths the stimulus digest already covers.
+        // An orchestration scenario's subagent tool lives in these files: editing one
+        // changes what the model could do without changing a single character of the
+        // spec, which is precisely the drift the staleness gate exists to catch.
+        for (const ext of s.extensions ?? []) {
+            if (ext in hashes)
+                continue;
+            hashes[ext] = fileSha256(resolve(ctx.specDir, ext)) ?? UNREADABLE;
+        }
         // The post-test IS the gate on a post_test scenario, and it lives outside the
         // fixture tree by convention (`fixture: fixtures/A1`, `post_test: post/A1.test.ts`),
         // so neither the fixture digest nor the scenario digest — which holds only the
@@ -425,6 +460,8 @@ export function scenarioSourceKeys(s) {
         keys.push(GATES_PREFIX + s.id);
     if (s.systemPromptFile)
         keys.push(s.systemPromptFile); // the agent file IS the stimulus
+    for (const ext of s.extensions ?? [])
+        keys.push(ext); // an edited extension is new stimulus
     if (s.assert?.post_test)
         keys.push(s.assert.post_test); // its contents are the gate
     const fx = effectiveFixture(s);

package/dist/spec-write.d.ts

@@ -0,0 +1,62 @@
+/**
+ * The single choke point for appending a scenario to an existing
+ * `specification.yaml`.
+ *
+ * Two callers need this — `add-test` and capture promotion — and a second
+ * implementation is how they would drift into disagreeing about what a valid
+ * write is. Everything here is deliberately append-shaped: a spec is
+ * hand-authored and full of comments, and a round trip through
+ * `yaml.load`/`yaml.dump` would silently reformat it and drop every comment the
+ * author wrote. So the existing bytes are never re-serialized — the new block is
+ * concatenated onto them and the *result* is validated before anything is
+ * written.
+ */
+/** Thrown when the spec on disk moved between the caller reading it and writing. */
+export declare class ConcurrentSpecModification extends Error {
+    constructor(specPath: string);
+}
+/** Thrown when the scenario being appended collides with one already in the spec. */
+export declare class DuplicateScenarioId extends Error {
+    constructor(id: string, specPath: string);
+}
+/** SHA-256 of spec text. Callers hold one across a read→confirm→write cycle. */
+export declare function specSha256(text: string): string;
+/**
+ * Render one scenario as a YAML block that can be concatenated onto a spec.
+ *
+ * Dumps `{ scenarios: [scenario] }` and strips the top-level key, leaving the
+ * correctly-indented list item. Going through `yaml.dump` rather than string
+ * templating is what makes arbitrary user text — quotes, colons, newlines,
+ * leading dashes — safe to embed.
+ */
+export declare function renderScenarioBlock(scenario: Record<string, unknown>): string;
+export interface AppendScenarioOptions {
+    specPath: string;
+    /** Plain object in spec field order; serialized by `renderScenarioBlock`. */
+    scenario: Record<string, unknown>;
+    /**
+     * SHA-256 the caller last saw. When supplied and the file no longer matches,
+     * the append is refused rather than layered onto someone else's edit.
+     */
+    baseSha256?: string;
+}
+export interface AppendScenarioResult {
+    id: string;
+    /** SHA-256 of the spec AFTER the append — the caller's new baseline. */
+    sha256: string;
+    /** The block that was appended, for preview/echo. */
+    block: string;
+}
+/**
+ * Validate and atomically append a scenario.
+ *
+ * Order matters and is load-bearing: read → detect concurrent modification →
+ * reject duplicate id → build → **validate the merged text** → write. The
+ * validation is on the merged result, not the block alone, because a block that
+ * parses in isolation can still break the file it lands in.
+ *
+ * The write is temp-file-plus-rename rather than `appendFileSync`. An append
+ * interrupted partway through leaves a syntactically broken spec on disk; a
+ * rename either happened or did not.
+ */
+export declare function appendScenario(opts: AppendScenarioOptions): AppendScenarioResult;