@skill-harness/core 0.21.0 → 0.22.0

package/dist/regate.d.ts

@@ -26,10 +26,12 @@ export interface RegateResult {
     judgeCalls: number;
 }
 /**
- * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
- * verdicts they determined — without re-running the model.
+ * Re-evaluate diff, output, and trace gates against a run's saved artifacts and
+ * re-decide the verdicts they determined — without re-running the model.
  *
- * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and
+ * `output_matches` / `output_excludes` are pure functions of the saved final
+ * assistant message. Since
  * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
  * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
  * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
@@ -48,6 +50,9 @@ export interface RegateResult {
  * That third row is what keeps this cheap without guessing: a rep the judge already
  * saw has its verdict on disk, so regate re-reads it rather than re-asking.
  *
+ * Output gates likewise use the complete saved transcript set and never re-run the
+ * subject. A newly failing output gate is therefore judge-free.
+ *
  * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
  * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
  * so a scenario carrying either is not regatable. Diffs and judge-raw files are

package/dist/regate.js

@@ -1,11 +1,11 @@
 import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
 import { parseVerdict, detectMisfire } from "./grade.js";
-import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
+import { evaluateNeedleGates, evaluateOutputGates, hasNeedleGates, hasOutputGates } from "./seeded.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
+import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, findDiffFiles, findTranscriptFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -13,6 +13,19 @@ import { gatesDigest, GATES_PREFIX } from "./sources.js";
 const GATE_FAILED_RE = /: (MISSING|PRESENT)$/m;
 const TRAILER = "=== SEEDED GATES ===";
 const DIFF_HEADER = "=== STAGED DIFF ===";
+const TRAILER_LINE_RE = /^=== SEEDED GATES ===[ \t]*$/gm;
+const DIFF_HEADER_LINE_RE = /^=== STAGED DIFF ===[ \t]*$/gm;
+/** Last exact full-line marker before `before`; model/diff substrings do not count. */
+function lastLineIndexBefore(text, pattern, before = text.length) {
+    pattern.lastIndex = 0;
+    let found = -1;
+    for (const match of text.matchAll(pattern)) {
+        if (match.index >= before)
+            break;
+        found = match.index;
+    }
+    return found;
+}
 /**
  * Rebuild a transcript with a fresh gates trailer, preserving the model's turns and
  * the embedded diff exactly.
@@ -24,10 +37,13 @@ const DIFF_HEADER = "=== STAGED DIFF ===";
  */
 function rewriteTranscript(path, gateLines) {
     const original = readFileSync(path, "utf8");
-    const trailerAt = original.indexOf(TRAILER);
+    // The harness-owned diff header is last; the harness-owned trailer is the last
+    // exact trailer line before it. Prefixes and marker text inside the diff do not
+    // become structural boundaries.
+    const diffAt = lastLineIndexBefore(original, DIFF_HEADER_LINE_RE);
+    const trailerAt = lastLineIndexBefore(original, TRAILER_LINE_RE, diffAt === -1 ? original.length : diffAt);
     if (trailerAt === -1)
         return; // no trailer to correct (non-seeded shape); leave it alone
-    const diffAt = original.indexOf(DIFF_HEADER);
     const head = original.slice(0, trailerAt);
     const tail = diffAt === -1 ? "" : original.slice(diffAt);
     renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
@@ -43,10 +59,12 @@ function verdictFromSavedJudgement(runDir, id, mode, rep) {
     return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
 }
 /**
- * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
- * verdicts they determined — without re-running the model.
+ * Re-evaluate diff, output, and trace gates against a run's saved artifacts and
+ * re-decide the verdicts they determined — without re-running the model.
  *
- * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and
+ * `output_matches` / `output_excludes` are pure functions of the saved final
+ * assistant message. Since
  * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
  * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
  * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
@@ -65,6 +83,9 @@ function verdictFromSavedJudgement(runDir, id, mode, rep) {
  * That third row is what keeps this cheap without guessing: a rep the judge already
  * saw has its verdict on disk, so regate re-reads it rather than re-asking.
  *
+ * Output gates likewise use the complete saved transcript set and never re-run the
+ * subject. A newly failing output gate is therefore judge-free.
+ *
  * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
  * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
  * so a scenario carrying either is not regatable. Diffs and judge-raw files are
@@ -87,8 +108,9 @@ export async function regateRun(opts) {
     for (const rec of prev.scenarios) {
         const s = specById.get(rec.id);
         const needles = hasNeedleGates(s ?? {});
+        const outputGated = hasOutputGates(s ?? {});
         const traceGated = Boolean(s?.traceAssert);
-        if (!s || (!needles && !traceGated))
+        if (!s || (!needles && !outputGated && !traceGated))
             continue; // nothing for regate to re-decide
         if (s.assert?.vitest || s.assert?.post_test) {
             blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
@@ -102,6 +124,10 @@ export async function regateRun(opts) {
             blocked.push(`${s.id}: staged-diff artifacts are incomplete for ${expectedReps} recorded rep(s) — regate refuses a partial repetition set`);
             continue;
         }
+        if (outputGated && !complete(findTranscriptFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => transcriptPath(opts.runDir, s.id, mode, rep)))) {
+            blocked.push(`${s.id}: transcript artifacts are incomplete for ${expectedReps} recorded rep(s) — regate refuses a partial repetition set`);
+            continue;
+        }
         // A trace gate is only re-decidable from a saved trace. A run recorded before
         // traces existed has none, and saying so is the whole point — pretending
         // regate can answer would report a verdict derived from no evidence.
@@ -114,7 +140,7 @@ export async function regateRun(opts) {
     }
     if (targets.length === 0) {
         throw new Error(`nothing to regate in ${opts.runDir}` +
-            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff_contains/diff_excludes or assert.trace"));
+            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff/output assertions or assert.trace"));
     }
     const changes = [];
     let judgeCalls = 0;
@@ -125,11 +151,11 @@ export async function regateRun(opts) {
             scenarios.push(rec); // untouched: not regatable, or no gates
             continue;
         }
-        const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
-        const traceFiles = findTraceFiles(opts.runDir, scenario.id, mode);
-        const repKeys = diffFiles.length > 0
-            ? diffFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: f }))
-            : traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
+        const expectedReps = rec.reps ?? 1;
+        const repKeys = (expectedReps === 1 ? [undefined] : Array.from({ length: expectedReps }, (_, index) => index)).map((rep) => ({
+            rep,
+            diffFile: hasNeedleGates(scenario) ? basename(diffPath(opts.runDir, scenario.id, mode, rep)) : undefined,
+        }));
         const outcomes = [];
         // Per scenario, not run-wide: with several regated scenarios, a global counter
         // would report every change as "re-judged" because some other scenario was.
@@ -139,9 +165,24 @@ export async function regateRun(opts) {
         for (const { rep, diffFile } of repKeys) {
             const diff = diffFile ? readFileSync(join(opts.runDir, diffFile), "utf8") : "";
             const needleGate = diffFile ? evaluateNeedleGates(scenario, diff) : { lines: [], failure: null };
-            // Trace gate, re-decided from the saved trace. Free: no model, no judge.
+            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
+            const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
+            const outputGate = evaluateOutputGates(scenario, before);
+            // Output and trace gates are re-decided from saved artifacts. Free: no model, no judge.
             let traceFailure = null;
-            let objective;
+            const expectedOutputHash = rec.objective?.rep_output_sha256?.[rep ?? 0]
+                ?? (rec.reps === undefined ? rec.objective?.output_sha256 : undefined);
+            const outputDigestMismatch = expectedOutputHash !== undefined && expectedOutputHash !== outputGate.outputSha256;
+            const outputIntegrityAssertion = outputDigestMismatch
+                ? [{ kind: "output_evidence", status: "ERROR", detail: "saved final assistant message no longer matches the hash recorded by the run" }]
+                : [];
+            let objective = hasOutputGates(scenario)
+                ? {
+                    status: outputDigestMismatch ? "ERROR" : outputGate.status,
+                    ...(outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {}),
+                    assertions: [...outputIntegrityAssertion, ...outputGate.assertions],
+                }
+                : undefined;
             if (scenario.traceAssert) {
                 const tp = tracePath(opts.runDir, scenario.id, mode, rep);
                 // A PARTIAL read is refused, not graded. `deserializeTrace` returns null
@@ -163,7 +204,7 @@ export async function regateRun(opts) {
                         usable.length === lines.length
                             ? "objective: saved trace is missing or unreadable — cannot re-evaluate assert.trace"
                             : `objective: saved trace is incomplete (${usable.length}/${lines.length} turns readable) — cannot re-evaluate assert.trace`;
-                    objective = { status: "ERROR", assertions: [] };
+                    objective = { ...(objective ?? {}), status: "ERROR", assertions: objective?.assertions ?? [] };
                 }
                 else {
                     const g = evaluateTraceGates(scenario.traceAssert, merged);
@@ -172,7 +213,17 @@ export async function regateRun(opts) {
                     const integrityAssertion = digestMismatch
                         ? [{ kind: "trace_evidence", status: "ERROR", detail: "saved trace no longer matches the hash recorded by the run" }]
                         : [];
-                    objective = { status: digestMismatch ? "ERROR" : g.status, trace_version: merged.trace_version, trace_sha256: merged.trace_sha256, assertions: [...integrityAssertion, ...g.assertions] };
+                    const priorStatus = objective?.status ?? "PASS";
+                    const status = digestMismatch || g.status === "ERROR" || priorStatus === "ERROR"
+                        ? "ERROR"
+                        : g.status === "FAIL" || priorStatus === "FAIL"
+                            ? "FAIL"
+                            : "PASS";
+                    objective = {
+                        ...(objective ?? { assertions: [] }), status,
+                        trace_version: merged.trace_version, trace_sha256: merged.trace_sha256,
+                        assertions: [...(objective?.assertions ?? []), ...integrityAssertion, ...g.assertions],
+                    };
                     if (g.status === "FAIL" || g.status === "ERROR" || digestMismatch) {
                         const bad = [...integrityAssertion.map((x) => x.detail), ...g.assertions.filter((x) => x.status === g.status && g.status !== "PASS").map((x) => x.detail)];
                         traceFailure = `objective: ${bad.join("; ")}`;
@@ -194,16 +245,29 @@ export async function regateRun(opts) {
                 if (delivery.status !== "PASS")
                     traceFailure = `objective: ${delivery.detail}`;
             }
-            const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure };
-            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
-            const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
+            const outputFailure = outputDigestMismatch
+                ? "objective: saved final assistant message no longer matches the hash recorded by the run"
+                : outputGate.status === "ERROR" || outputGate.status === "FAIL" ? outputGate.failure : null;
+            const gate = { lines: [...needleGate.lines, ...outputGate.lines], failure: needleGate.failure ?? outputFailure ?? traceFailure };
             // Two sources, because the two gate kinds record their prior state
             // differently: a seeded needle gate leaves a trailer in the transcript, a
             // trace gate leaves an `objective` block on the result. Reading only the
             // trailer meant a trace gate flipping to PASS never triggered the re-judge
             // it needs, leaving a stale FAIL verdict beside a PASS objective.
-            const oldObjectiveFailed = rec.objective?.status === "FAIL" || rec.objective?.status === "ERROR" || rec.objective?.status === "NOT-MEASURED";
-            const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER))) || oldObjectiveFailed;
+            const perRepObjective = rec.rep_judgments?.find(panel => panel.repetition === (rep ?? 0))?.objective;
+            // Modern --reps records carry objective evidence per repetition. Falling
+            // back to the strict scenario aggregate there would label every rep failed
+            // when only one failed, and unnecessarily re-judge already-passing reps.
+            // Legacy records without panels have only the aggregate, so retain that
+            // conservative fallback for compatibility.
+            const oldObjective = perRepObjective ?? (rec.rep_judgments?.length ? undefined : rec.objective);
+            const oldObjectiveFailed = oldObjective?.status === "FAIL" || oldObjective?.status === "ERROR" || oldObjective?.status === "NOT-MEASURED";
+            const oldDiffAt = lastLineIndexBefore(before, DIFF_HEADER_LINE_RE);
+            const oldTrailerAt = lastLineIndexBefore(before, TRAILER_LINE_RE, oldDiffAt === -1 ? before.length : oldDiffAt);
+            const oldGateTrailer = oldTrailerAt === -1
+                ? ""
+                : before.slice(oldTrailerAt, oldDiffAt === -1 ? before.length : oldDiffAt);
+            const oldGateFailed = GATE_FAILED_RE.test(oldGateTrailer) || oldObjectiveFailed;
             // The trailer is regenerated whatever the outcome: leaving a stale
             // `MISSING` note beside a corrected verdict would misinform the next reader
             // (and the next judge, which reads this transcript).

package/dist/reps.js

@@ -16,9 +16,11 @@ export function aggregateObjective(outcomes) {
     if (present.length === 1)
         return picked;
     const traceHashes = present.map((objective) => objective.trace_sha256);
+    const outputHashes = present.map((objective) => objective.output_sha256);
     return {
         ...picked,
         ...(traceHashes.every((hash) => typeof hash === "string") ? { rep_trace_sha256: traceHashes } : {}),
+        ...(outputHashes.every((hash) => typeof hash === "string") ? { rep_output_sha256: outputHashes } : {}),
     };
 }
 /**

package/dist/results.d.ts

@@ -117,8 +117,11 @@ export interface ObjectiveResult {
     status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
     trace_version?: number;
     trace_sha256?: string;
+    /** SHA-256 of the final assistant message evaluated by output regex gates. */
+    output_sha256?: string;
     /** Per-repetition hashes retained when an aggregate has more than one rep. */
     rep_trace_sha256?: string[];
+    rep_output_sha256?: string[];
     assertions: {
         kind: string;
         status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";

package/dist/run.js

@@ -6,7 +6,7 @@ import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
-import { runSeeded } from "./seeded.js";
+import { evaluateOutputGates, hasOutputGates, runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
@@ -194,11 +194,11 @@ export async function runSkillModel(opts) {
  * ends the last assistant section.
  */
 export function hasEmptyAssistantTurn(transcript) {
-    const sections = transcript.split(/^<<< ASSISTANT:\s*$/m).slice(1);
+    const sections = transcript.split(/^<<< ASSISTANT:[ \t]*$/m).slice(1);
     if (sections.length === 0)
         return false;
     return sections.some((sec) => {
-        const body = sec.split(/^(?:>>> |=== SEEDED GATES ===|\[pi exited )/m)[0];
+        const body = sec.split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*|\[pi exited [^\]]+\][ \t]*)$/m)[0];
         return body.trim() === "";
     });
 }
@@ -396,6 +396,9 @@ async function runRep(scenario, rep, repCount, ctx) {
             }
         }
         const repSuffix = repCount > 1 ? rep : undefined;
+        const outputGate = !adapterFailure && !noResponse && !infrastructureFailure
+            ? evaluateOutputGates(scenario, transcript)
+            : { status: "PASS", failure: null, assertions: [], lines: [] };
         writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
         if (scenario.mode === "seeded") {
             // The workspace is torn down in the `finally` below, so this is the only
@@ -440,10 +443,11 @@ async function runRep(scenario, rep, repCount, ctx) {
         // timeout that caused it, and it is that text, not the real reason, that would
         // reach the results record. The verdict is ERROR either way.
         let objective;
-        if (scenario.traceAssert && !adapterFailure) {
-            const assertionResults = [];
-            let status = "PASS";
+        if ((scenario.traceAssert || hasOutputGates(scenario)) && !adapterFailure && !noResponse && !infrastructureFailure) {
+            const assertionResults = [...outputGate.assertions];
+            let status = outputGate.status;
             let traceMeta = {};
+            const outputMeta = outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {};
             if (scenario.traceAssert) {
                 if (traces.length > 0) {
                     writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
@@ -459,12 +463,15 @@ async function runRep(scenario, rep, repCount, ctx) {
                 }
                 else {
                     const gate = evaluateTraceGates(scenario.traceAssert, merged);
-                    status = gate.status;
+                    if (gate.status === "ERROR" || status === "ERROR")
+                        status = "ERROR";
+                    else if (gate.status === "FAIL" || status === "FAIL")
+                        status = "FAIL";
                     assertionResults.push(...gate.assertions);
                     traceMeta = { trace_version: merged.trace_version, trace_sha256: merged.trace_sha256 };
                 }
             }
-            objective = { status, ...traceMeta, assertions: assertionResults };
+            objective = { status, ...traceMeta, ...outputMeta, assertions: assertionResults };
             if (status !== "PASS") {
                 const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
                 gatePrefix = `objective: ${details.join("; ") || "structured evidence could not be evaluated"}`;

package/dist/seeded.d.ts

@@ -82,6 +82,28 @@ export interface SeededOutcome {
 export declare function changedLines(diff: string): string;
 /** Whether a scenario declares any needle gate at all. */
 export declare function hasNeedleGates(scenario: Scenario): boolean;
+/** Whether a scenario declares any final-assistant-message gate. */
+export declare function hasOutputGates(scenario: Scenario): boolean;
+/**
+ * The final assistant turn exactly as the transcript artifact records its text.
+ *
+ * Delimiter-shaped model output is ambiguous in this plain-text artifact. Refuse
+ * it instead of truncating the output and turning a forbidden suffix into PASS.
+ */
+export declare function finalAssistantMessage(transcript: string, scenario: Scenario): string | null;
+export interface OutputGateResult {
+    status: "PASS" | "FAIL" | "ERROR";
+    failure: string | null;
+    outputSha256?: string;
+    assertions: {
+        kind: "output_matches" | "output_excludes" | "output_evidence";
+        status: "PASS" | "FAIL" | "ERROR";
+        detail: string;
+    }[];
+    lines: string[];
+}
+/** Evaluate final-message regex gates without exposing the matched text. */
+export declare function evaluateOutputGates(scenario: Scenario, transcript: string): OutputGateResult;
 /**
  * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
  * to report, and the first failure (null when every needle is satisfied).

package/dist/seeded.js

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { copyFileSync, statSync } from "node:fs";
 import { extname, isAbsolute, join, resolve } from "node:path";
 import { exec } from "./util/exec.js";
@@ -60,6 +61,74 @@ export function changedLines(diff) {
 export function hasNeedleGates(scenario) {
     return (scenario.assert?.diff_contains?.length ?? 0) > 0 || (scenario.assert?.diff_excludes?.length ?? 0) > 0;
 }
+/** Whether a scenario declares any final-assistant-message gate. */
+export function hasOutputGates(scenario) {
+    return (scenario.assert?.output_matches?.length ?? 0) > 0 || (scenario.assert?.output_excludes?.length ?? 0) > 0;
+}
+/**
+ * The final assistant turn exactly as the transcript artifact records its text.
+ *
+ * Delimiter-shaped model output is ambiguous in this plain-text artifact. Refuse
+ * it instead of truncating the output and turning a forbidden suffix into PASS.
+ */
+export function finalAssistantMessage(transcript, scenario) {
+    const assistantMarker = /^<<< ASSISTANT:[ \t]*$/gm;
+    const userMarker = /^>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*$/gm;
+    const gateMarker = /^=== SEEDED GATES ===[ \t]*$/gm;
+    const exitMarker = /^\[pi exited [^\]]+\][ \t]*$/gm;
+    const assistantCount = [...transcript.matchAll(assistantMarker)].length;
+    const userCount = [...transcript.matchAll(userMarker)].length;
+    const gateCount = [...transcript.matchAll(gateMarker)].length;
+    const exitCount = [...transcript.matchAll(exitMarker)].length;
+    const expectedGates = scenario.mode === "seeded" ? 1 : 0;
+    if (assistantCount !== scenario.turns.length || userCount !== scenario.turns.length || gateCount !== expectedGates || exitCount !== 0) {
+        return null;
+    }
+    const sections = transcript.split(/^<<< ASSISTANT:[ \t]*$/m);
+    return sections[sections.length - 1]
+        .split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*|\[pi exited [^\]]+\][ \t]*)$/m)[0]
+        .trim();
+}
+/** Evaluate final-message regex gates without exposing the matched text. */
+export function evaluateOutputGates(scenario, transcript) {
+    if (!hasOutputGates(scenario))
+        return { status: "PASS", failure: null, assertions: [], lines: [] };
+    const output = finalAssistantMessage(transcript, scenario);
+    if (output === null) {
+        const detail = "final assistant message is missing or transcript delimiters are ambiguous";
+        return {
+            status: "ERROR", failure: `output evidence: ${detail}`,
+            assertions: [{ kind: "output_evidence", status: "ERROR", detail }],
+            lines: [`  output: ERROR (${detail})`],
+        };
+    }
+    const assertions = [];
+    const lines = [];
+    let failure = null;
+    for (const pattern of scenario.assert?.output_matches ?? []) {
+        const ok = new RegExp(pattern, "m").test(output);
+        const detail = ok ? `final assistant message matches ${JSON.stringify(pattern)}` : `final assistant message does not match ${JSON.stringify(pattern)}`;
+        assertions.push({ kind: "output_matches", status: ok ? "PASS" : "FAIL", detail });
+        lines.push(`  output_matches ${JSON.stringify(pattern)}: ${ok ? "MATCH" : "MISSING"}`);
+        if (!ok && !failure)
+            failure = `output_matches ${JSON.stringify(pattern)} did not match the final assistant message`;
+    }
+    for (const pattern of scenario.assert?.output_excludes ?? []) {
+        const ok = !new RegExp(pattern, "m").test(output);
+        const detail = ok ? `final assistant message excludes ${JSON.stringify(pattern)}` : `final assistant message matches forbidden regex ${JSON.stringify(pattern)}`;
+        assertions.push({ kind: "output_excludes", status: ok ? "PASS" : "FAIL", detail });
+        lines.push(`  output_excludes ${JSON.stringify(pattern)}: ${ok ? "ABSENT" : "PRESENT"}`);
+        if (!ok && !failure)
+            failure = `output_excludes ${JSON.stringify(pattern)} matched the final assistant message`;
+    }
+    return {
+        status: failure ? "FAIL" : "PASS",
+        failure,
+        outputSha256: createHash("sha256").update(output).digest("hex"),
+        assertions,
+        lines,
+    };
+}
 /**
  * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
  * to report, and the first failure (null when every needle is satisfied).

package/dist/sources.d.ts

@@ -54,7 +54,7 @@ export declare const FIXTURE_PREFIX = "fixture:";
  * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
  * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
  * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
- * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
+ * | `gates:<id>` | diff/output regex gates and `assert.trace` | gate wrong, saved evidence still valid | `regate` (no subject call; judges fail→pass reps) |
  * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
  *
  * Why this matters more than it looks: with one key, lint had exactly one remedy for
@@ -105,7 +105,7 @@ export declare function dirSha256(dir: string): string | null;
 export declare function stimulusDigest(s: Scenario): string;
 export declare function rubricDigest(s: Scenario): string;
 export declare function policyDigest(s: Scenario): string;
-/** Null when the scenario declares no needle gates — no key is recorded for it. */
+/** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export declare function gatesDigest(s: Scenario): string | null;
 /** The spec-level judge persona, which is rubric for every scenario at once. */
 export declare function personaDigest(persona: string): string;

package/dist/sources.js

@@ -58,7 +58,7 @@ export const FIXTURE_PREFIX = "fixture:";
  * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
  * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
  * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
- * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
+ * | `gates:<id>` | diff/output regex gates and `assert.trace` | gate wrong, saved evidence still valid | `regate` (no subject call; judges fail→pass reps) |
  * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
  *
  * Why this matters more than it looks: with one key, lint had exactly one remedy for
@@ -163,7 +163,7 @@ function walk(dir, prefix = "") {
  * A scenario's fields, sorted into the four buckets, as canonical JSON.
  *
  * One function so the exhaustive-destructure trick below covers all four digests at
- * once: adding a field to `Scenario` or `SeededAssert` fails the build **here** until
+ * once: adding a field to `Scenario` or `ScenarioAssert` fails the build **here** until
  * someone decides which bucket — and therefore which remedy — it belongs to. A field
  * nobody assigned is a permanent staleness blind spot, and a field assigned to the
  * wrong bucket is worse than that: it would tell a user `rescore` is enough when the
@@ -180,14 +180,14 @@ function facets(s) {
     // anywhere would charge a re-run (or at best a re-score) for editing a label,
     // which is the exact trap the facet split was built to remove.
     void _coversIsMetadata;
-    const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
+    const { vitest, diff_contains, diff_excludes, output_matches, output_excludes, post_test, ...restAssert } = assert ?? {};
     const _assertExhaustive = restAssert;
     void _assertExhaustive;
     // `traceAssert` is a GATE, not stimulus: it is evaluated against a trace the run
     // already saved, so `regate` can re-answer it without re-running the subject model.
     // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
     // changes what gets executed, the other only what we conclude from it.
-    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || output_matches !== undefined || output_excludes !== undefined || traceAssert !== undefined;
     return {
         // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
         // run executes in the workspace, and neither can be re-evaluated from a saved
@@ -214,6 +214,8 @@ function facets(s) {
         gates: hasGates
             ? JSON.stringify([
                 id, diff_contains ?? null, diff_excludes ?? null,
+                ...(output_matches ? [["output_matches", output_matches]] : []),
+                ...(output_excludes ? [["output_excludes", output_excludes]] : []),
                 ...(traceAssert ? [traceAssert] : []),
             ])
             : null,
@@ -231,7 +233,7 @@ export function rubricDigest(s) {
 export function policyDigest(s) {
     return sha(facets(s).policy);
 }
-/** Null when the scenario declares no needle gates — no key is recorded for it. */
+/** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export function gatesDigest(s) {
     const g = facets(s).gates;
     return g === null ? null : sha(g);

package/dist/spec.d.ts

@@ -1,7 +1,7 @@
 import type { WorkspaceKind } from "./workspace.js";
 import { type TraceAssert } from "./trace-gates.js";
 export type ScenarioMode = "inline" | "seeded";
-export interface SeededAssert {
+export interface ScenarioAssert {
     vitest?: boolean;
     diff_contains?: string[];
     /**
@@ -11,6 +11,10 @@ export interface SeededAssert {
      * from the model's prose.
      */
     diff_excludes?: string[];
+    /** Regular expressions that must all match the final assistant message. */
+    output_matches?: string[];
+    /** Regular expressions that must not match the final assistant message. */
+    output_excludes?: string[];
     /**
      * A test file copied into the workspace AFTER the agent finishes, then run.
      * The model never sees it, so it cannot write code shaped to pass it — this
@@ -28,12 +32,12 @@ export interface Scenario {
     turns: string[];
     checklist: string[];
     fixture?: string;
-    assert?: SeededAssert;
+    assert?: ScenarioAssert;
     /**
      * Objective assertions over the execution trace.
      *
-     * Deliberately NOT part of `SeededAssert`: the other gates read a staged git
-     * diff and are meaningless without a fixture, while a trace exists for any run.
+     * Deliberately NOT part of `ScenarioAssert`: trace assertions have their own
+     * closed schema and opt execution into structured capture.
      * Declaring it opts the scenario into structured (`--mode json`) execution.
      */
     traceAssert?: TraceAssert;

package/dist/spec.js

@@ -11,6 +11,21 @@ export class SpecError extends Error {
 function isStringArray(v) {
     return Array.isArray(v) && v.every((x) => typeof x === "string");
 }
+function parseRegexList(v, id, field, file) {
+    assertStringList(v, id, field, file);
+    for (const pattern of v) {
+        if (pattern === "") {
+            throw new SpecError(`scenario \`${id}\` \`${field}\` patterns must not be empty`, file);
+        }
+        try {
+            new RegExp(pattern, "m");
+        }
+        catch (e) {
+            throw new SpecError(`scenario \`${id}\` \`${field}\` contains invalid regular expression ${JSON.stringify(pattern)} — ${e.message}`, file);
+        }
+    }
+    return v;
+}
 /**
  * Require a non-empty list of strings, with a targeted error. A common authoring
  * trap: an unquoted "key: value" list item parses as a YAML mapping, not a string
@@ -176,23 +191,39 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
-        // `assert.trace` is legal for inline AND seeded scenarios — it reads the
-        // execution trace, which every run produces, not a staged diff.
+        // Output and trace assertions are legal for inline AND seeded scenarios.
+        if (s.assert !== undefined && (s.assert === null || typeof s.assert !== "object" || Array.isArray(s.assert))) {
+            throw new SpecError(`scenario \`${id}\` \`assert\` must be a mapping`, file);
+        }
         const rawAssert = s.assert;
         if (rawAssert?.trajectory !== undefined) {
             throw new SpecError(`scenario \`${id}\` uses removed \`assert.trajectory\`; delete it or replace it with an active objective gate`, file);
         }
+        const allowedAssertKeys = new Set([
+            "vitest", "diff_contains", "diff_excludes", "post_test", "trace", "output_matches", "output_excludes",
+        ]);
+        for (const key of Object.keys(rawAssert ?? {})) {
+            if (!allowedAssertKeys.has(key)) {
+                throw new SpecError(`scenario \`${id}\` has unknown \`assert\` key \`${key}\``, file);
+            }
+        }
         if (rawAssert?.trace !== undefined) {
             scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
         }
+        const assertObj = {};
+        if (rawAssert?.output_matches !== undefined) {
+            assertObj.output_matches = parseRegexList(rawAssert.output_matches, id, "assert.output_matches", file);
+        }
+        if (rawAssert?.output_excludes !== undefined) {
+            assertObj.output_excludes = parseRegexList(rawAssert.output_excludes, id, "assert.output_excludes", file);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
             }
             scenario.fixture = s.fixture;
-            const a = s.assert;
+            const a = rawAssert;
             if (a) {
-                const assertObj = {};
                 if (a.vitest !== undefined)
                     assertObj.vitest = a.vitest === true;
                 if (a.diff_contains !== undefined) {
@@ -232,9 +263,10 @@ export function parseSpec(text, file) {
                     }
                     assertObj.post_test = a.post_test.trim();
                 }
-                scenario.assert = assertObj;
             }
         }
+        if (Object.keys(assertObj).length > 0)
+            scenario.assert = assertObj;
         if (s.env && typeof s.env === "object" && Object.hasOwn(s.env, "event_sources")) {
             throw new SpecError(`scenario \`${id}\` uses removed \`env.event_sources\`; it is no longer collected`, file);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.21.0",
+    "version": "0.22.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -353,6 +353,10 @@
             "trace_sha256": {
               "type": "string"
             },
+            "output_sha256": {
+              "type": "string",
+              "pattern": "^[a-fA-F0-9]{64}$"
+            },
             "rep_trace_sha256": {
               "type": "array",
               "items": {
@@ -360,6 +364,13 @@
                 "pattern": "^[a-fA-F0-9]{64}$"
               }
             },
+            "rep_output_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
+            },
             "assertions": {
               "type": "array",
               "items": {

package/schemas/results-v3.schema.json

@@ -380,6 +380,10 @@
             "trace_sha256": {
               "type": "string"
             },
+            "output_sha256": {
+              "type": "string",
+              "pattern": "^[a-fA-F0-9]{64}$"
+            },
             "trajectory_version": {
               "const": "1.0"
             },
@@ -400,6 +404,13 @@
                 "pattern": "^[a-fA-F0-9]{64}$"
               }
             },
+            "rep_output_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
+            },
             "assertions": {
               "type": "array",
               "items": {

package/schemas/specification-v1.schema.json

@@ -157,6 +157,22 @@
                 "minLength": 1
               }
             },
+            "output_matches": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "output_excludes": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
             "post_test": {
               "type": "string"
             },