@skill-harness/core 0.21.0 → 0.22.3

package/dist/grade.d.ts

@@ -38,6 +38,8 @@ export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef
 export interface GradeResult extends ParsedVerdict {
     raw: string;
     criteria: CriterionVote[];
+    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
@@ -50,10 +52,10 @@ export interface GradeResult extends ParsedVerdict {
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
 /** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string): Promise<GradeResult>;
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes } from "./results.js";
+import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -123,8 +123,16 @@ export function detectMisfire(raw, verdict) {
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
 /** Drive the judge for one transcript and parse the result. */
-export async function gradeTranscript(adapter, judge, prompt, cwd) {
-    const raw = await adapter.judge({ model: judge, prompt, cwd });
+export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
+    const request = { model: judge, prompt, cwd };
+    let raw = await adapter.judge(request);
+    let criteria = parseCriterionVotes(raw);
+    let judgeRetries;
+    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
+        raw = await adapter.judge(request);
+        criteria = parseCriterionVotes(raw);
+        judgeRetries = 1;
+    }
     const parsed = parseVerdict(raw);
     // On a parse failure, surface what the judge actually emitted (e.g. a provider
     // error) rather than a generic message — otherwise the cause is invisible.
@@ -134,17 +142,17 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
             parsed.reason = `judge unparseable: ${snippet}`;
     }
     const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
+    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export async function judgeInWorkspace(adapter, judge, prompt, specDir) {
+export async function judgeInWorkspace(adapter, judge, prompt, specDir, expectedCriteria) {
     const ws = createWorkspace("none", { specDir });
     try {
-        return await gradeTranscript(adapter, judge, prompt, ws.cwd);
+        return await gradeTranscript(adapter, judge, prompt, ws.cwd, expectedCriteria);
     }
     finally {
         ws.cleanup();

package/dist/regate.d.ts

@@ -26,10 +26,12 @@ export interface RegateResult {
     judgeCalls: number;
 }
 /**
- * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
- * verdicts they determined — without re-running the model.
+ * Re-evaluate diff, output, and trace gates against a run's saved artifacts and
+ * re-decide the verdicts they determined — without re-running the model.
  *
- * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and
+ * `output_matches` / `output_excludes` are pure functions of the saved final
+ * assistant message. Since
  * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
  * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
  * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
@@ -48,6 +50,9 @@ export interface RegateResult {
  * That third row is what keeps this cheap without guessing: a rep the judge already
  * saw has its verdict on disk, so regate re-reads it rather than re-asking.
  *
+ * Output gates likewise use the complete saved transcript set and never re-run the
+ * subject. A newly failing output gate is therefore judge-free.
+ *
  * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
  * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
  * so a scenario carrying either is not regatable. Diffs and judge-raw files are

package/dist/regate.js

@@ -1,11 +1,11 @@
 import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
 import { parseVerdict, detectMisfire } from "./grade.js";
-import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
+import { evaluateNeedleGates, evaluateOutputGates, hasNeedleGates, hasOutputGates } from "./seeded.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
+import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, findDiffFiles, findTranscriptFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -13,6 +13,19 @@ import { gatesDigest, GATES_PREFIX } from "./sources.js";
 const GATE_FAILED_RE = /: (MISSING|PRESENT)$/m;
 const TRAILER = "=== SEEDED GATES ===";
 const DIFF_HEADER = "=== STAGED DIFF ===";
+const TRAILER_LINE_RE = /^=== SEEDED GATES ===[ \t]*$/gm;
+const DIFF_HEADER_LINE_RE = /^=== STAGED DIFF ===[ \t]*$/gm;
+/** Last exact full-line marker before `before`; model/diff substrings do not count. */
+function lastLineIndexBefore(text, pattern, before = text.length) {
+    pattern.lastIndex = 0;
+    let found = -1;
+    for (const match of text.matchAll(pattern)) {
+        if (match.index >= before)
+            break;
+        found = match.index;
+    }
+    return found;
+}
 /**
  * Rebuild a transcript with a fresh gates trailer, preserving the model's turns and
  * the embedded diff exactly.
@@ -24,10 +37,13 @@ const DIFF_HEADER = "=== STAGED DIFF ===";
  */
 function rewriteTranscript(path, gateLines) {
     const original = readFileSync(path, "utf8");
-    const trailerAt = original.indexOf(TRAILER);
+    // The harness-owned diff header is last; the harness-owned trailer is the last
+    // exact trailer line before it. Prefixes and marker text inside the diff do not
+    // become structural boundaries.
+    const diffAt = lastLineIndexBefore(original, DIFF_HEADER_LINE_RE);
+    const trailerAt = lastLineIndexBefore(original, TRAILER_LINE_RE, diffAt === -1 ? original.length : diffAt);
     if (trailerAt === -1)
         return; // no trailer to correct (non-seeded shape); leave it alone
-    const diffAt = original.indexOf(DIFF_HEADER);
     const head = original.slice(0, trailerAt);
     const tail = diffAt === -1 ? "" : original.slice(diffAt);
     renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
@@ -43,10 +59,12 @@ function verdictFromSavedJudgement(runDir, id, mode, rep) {
     return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
 }
 /**
- * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
- * verdicts they determined — without re-running the model.
+ * Re-evaluate diff, output, and trace gates against a run's saved artifacts and
+ * re-decide the verdicts they determined — without re-running the model.
  *
- * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and
+ * `output_matches` / `output_excludes` are pure functions of the saved final
+ * assistant message. Since
  * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
  * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
  * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
@@ -65,6 +83,9 @@ function verdictFromSavedJudgement(runDir, id, mode, rep) {
  * That third row is what keeps this cheap without guessing: a rep the judge already
  * saw has its verdict on disk, so regate re-reads it rather than re-asking.
  *
+ * Output gates likewise use the complete saved transcript set and never re-run the
+ * subject. A newly failing output gate is therefore judge-free.
+ *
  * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
  * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
  * so a scenario carrying either is not regatable. Diffs and judge-raw files are
@@ -87,8 +108,9 @@ export async function regateRun(opts) {
     for (const rec of prev.scenarios) {
         const s = specById.get(rec.id);
         const needles = hasNeedleGates(s ?? {});
+        const outputGated = hasOutputGates(s ?? {});
         const traceGated = Boolean(s?.traceAssert);
-        if (!s || (!needles && !traceGated))
+        if (!s || (!needles && !outputGated && !traceGated))
             continue; // nothing for regate to re-decide
         if (s.assert?.vitest || s.assert?.post_test) {
             blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
@@ -102,6 +124,10 @@ export async function regateRun(opts) {
             blocked.push(`${s.id}: staged-diff artifacts are incomplete for ${expectedReps} recorded rep(s) — regate refuses a partial repetition set`);
             continue;
         }
+        if (outputGated && !complete(findTranscriptFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => transcriptPath(opts.runDir, s.id, mode, rep)))) {
+            blocked.push(`${s.id}: transcript artifacts are incomplete for ${expectedReps} recorded rep(s) — regate refuses a partial repetition set`);
+            continue;
+        }
         // A trace gate is only re-decidable from a saved trace. A run recorded before
         // traces existed has none, and saying so is the whole point — pretending
         // regate can answer would report a verdict derived from no evidence.
@@ -114,7 +140,7 @@ export async function regateRun(opts) {
     }
     if (targets.length === 0) {
         throw new Error(`nothing to regate in ${opts.runDir}` +
-            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff_contains/diff_excludes or assert.trace"));
+            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff/output assertions or assert.trace"));
     }
     const changes = [];
     let judgeCalls = 0;
@@ -125,11 +151,11 @@ export async function regateRun(opts) {
             scenarios.push(rec); // untouched: not regatable, or no gates
             continue;
         }
-        const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
-        const traceFiles = findTraceFiles(opts.runDir, scenario.id, mode);
-        const repKeys = diffFiles.length > 0
-            ? diffFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: f }))
-            : traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
+        const expectedReps = rec.reps ?? 1;
+        const repKeys = (expectedReps === 1 ? [undefined] : Array.from({ length: expectedReps }, (_, index) => index)).map((rep) => ({
+            rep,
+            diffFile: hasNeedleGates(scenario) ? basename(diffPath(opts.runDir, scenario.id, mode, rep)) : undefined,
+        }));
         const outcomes = [];
         // Per scenario, not run-wide: with several regated scenarios, a global counter
         // would report every change as "re-judged" because some other scenario was.
@@ -139,9 +165,24 @@ export async function regateRun(opts) {
         for (const { rep, diffFile } of repKeys) {
             const diff = diffFile ? readFileSync(join(opts.runDir, diffFile), "utf8") : "";
             const needleGate = diffFile ? evaluateNeedleGates(scenario, diff) : { lines: [], failure: null };
-            // Trace gate, re-decided from the saved trace. Free: no model, no judge.
+            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
+            const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
+            const outputGate = evaluateOutputGates(scenario, before);
+            // Output and trace gates are re-decided from saved artifacts. Free: no model, no judge.
             let traceFailure = null;
-            let objective;
+            const expectedOutputHash = rec.objective?.rep_output_sha256?.[rep ?? 0]
+                ?? (rec.reps === undefined ? rec.objective?.output_sha256 : undefined);
+            const outputDigestMismatch = expectedOutputHash !== undefined && expectedOutputHash !== outputGate.outputSha256;
+            const outputIntegrityAssertion = outputDigestMismatch
+                ? [{ kind: "output_evidence", status: "ERROR", detail: "saved final assistant message no longer matches the hash recorded by the run" }]
+                : [];
+            let objective = hasOutputGates(scenario)
+                ? {
+                    status: outputDigestMismatch ? "ERROR" : outputGate.status,
+                    ...(outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {}),
+                    assertions: [...outputIntegrityAssertion, ...outputGate.assertions],
+                }
+                : undefined;
             if (scenario.traceAssert) {
                 const tp = tracePath(opts.runDir, scenario.id, mode, rep);
                 // A PARTIAL read is refused, not graded. `deserializeTrace` returns null
@@ -163,7 +204,7 @@ export async function regateRun(opts) {
                         usable.length === lines.length
                             ? "objective: saved trace is missing or unreadable — cannot re-evaluate assert.trace"
                             : `objective: saved trace is incomplete (${usable.length}/${lines.length} turns readable) — cannot re-evaluate assert.trace`;
-                    objective = { status: "ERROR", assertions: [] };
+                    objective = { ...(objective ?? {}), status: "ERROR", assertions: objective?.assertions ?? [] };
                 }
                 else {
                     const g = evaluateTraceGates(scenario.traceAssert, merged);
@@ -172,7 +213,17 @@ export async function regateRun(opts) {
                     const integrityAssertion = digestMismatch
                         ? [{ kind: "trace_evidence", status: "ERROR", detail: "saved trace no longer matches the hash recorded by the run" }]
                         : [];
-                    objective = { status: digestMismatch ? "ERROR" : g.status, trace_version: merged.trace_version, trace_sha256: merged.trace_sha256, assertions: [...integrityAssertion, ...g.assertions] };
+                    const priorStatus = objective?.status ?? "PASS";
+                    const status = digestMismatch || g.status === "ERROR" || priorStatus === "ERROR"
+                        ? "ERROR"
+                        : g.status === "FAIL" || priorStatus === "FAIL"
+                            ? "FAIL"
+                            : "PASS";
+                    objective = {
+                        ...(objective ?? { assertions: [] }), status,
+                        trace_version: merged.trace_version, trace_sha256: merged.trace_sha256,
+                        assertions: [...(objective?.assertions ?? []), ...integrityAssertion, ...g.assertions],
+                    };
                     if (g.status === "FAIL" || g.status === "ERROR" || digestMismatch) {
                         const bad = [...integrityAssertion.map((x) => x.detail), ...g.assertions.filter((x) => x.status === g.status && g.status !== "PASS").map((x) => x.detail)];
                         traceFailure = `objective: ${bad.join("; ")}`;
@@ -194,16 +245,29 @@ export async function regateRun(opts) {
                 if (delivery.status !== "PASS")
                     traceFailure = `objective: ${delivery.detail}`;
             }
-            const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure };
-            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
-            const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
+            const outputFailure = outputDigestMismatch
+                ? "objective: saved final assistant message no longer matches the hash recorded by the run"
+                : outputGate.status === "ERROR" || outputGate.status === "FAIL" ? outputGate.failure : null;
+            const gate = { lines: [...needleGate.lines, ...outputGate.lines], failure: needleGate.failure ?? outputFailure ?? traceFailure };
             // Two sources, because the two gate kinds record their prior state
             // differently: a seeded needle gate leaves a trailer in the transcript, a
             // trace gate leaves an `objective` block on the result. Reading only the
             // trailer meant a trace gate flipping to PASS never triggered the re-judge
             // it needs, leaving a stale FAIL verdict beside a PASS objective.
-            const oldObjectiveFailed = rec.objective?.status === "FAIL" || rec.objective?.status === "ERROR" || rec.objective?.status === "NOT-MEASURED";
-            const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER))) || oldObjectiveFailed;
+            const perRepObjective = rec.rep_judgments?.find(panel => panel.repetition === (rep ?? 0))?.objective;
+            // Modern --reps records carry objective evidence per repetition. Falling
+            // back to the strict scenario aggregate there would label every rep failed
+            // when only one failed, and unnecessarily re-judge already-passing reps.
+            // Legacy records without panels have only the aggregate, so retain that
+            // conservative fallback for compatibility.
+            const oldObjective = perRepObjective ?? (rec.rep_judgments?.length ? undefined : rec.objective);
+            const oldObjectiveFailed = oldObjective?.status === "FAIL" || oldObjective?.status === "ERROR" || oldObjective?.status === "NOT-MEASURED";
+            const oldDiffAt = lastLineIndexBefore(before, DIFF_HEADER_LINE_RE);
+            const oldTrailerAt = lastLineIndexBefore(before, TRAILER_LINE_RE, oldDiffAt === -1 ? before.length : oldDiffAt);
+            const oldGateTrailer = oldTrailerAt === -1
+                ? ""
+                : before.slice(oldTrailerAt, oldDiffAt === -1 ? before.length : oldDiffAt);
+            const oldGateFailed = GATE_FAILED_RE.test(oldGateTrailer) || oldObjectiveFailed;
             // The trailer is regenerated whatever the outcome: leaving a stale
             // `MISSING` note beside a corrected verdict would misinform the next reader
             // (and the next judge, which reads this transcript).
@@ -226,12 +290,13 @@ export async function regateRun(opts) {
             }
             // The gate blocked this rep before, so no judgement of it exists anywhere.
             const transcript = readFileSync(tPath, "utf8");
-            outcomes.push({ ...(await judgeOneRep({
-                    runDir: opts.runDir, spec: opts.spec, scenario, transcript,
-                    adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                    mode, rep, now, rejudge: true,
-                })), objective });
-            judgeCalls++;
+            const judged = await judgeOneRep({
+                runDir: opts.runDir, spec: opts.spec, scenario, transcript,
+                adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
+                mode, rep, now, rejudge: true,
+            });
+            outcomes.push({ ...judged, objective });
+            judgeCalls += judged.metrics?.judge_calls ?? 0;
             judgedHere++;
         }
         const threshold = effectiveThreshold(rec, scenario);

package/dist/regrade.d.ts

@@ -34,6 +34,9 @@ export interface RegradeOptions {
     mode?: string;
     /** Recorded repetition count; when supplied, missing/extra rep artifacts fail before judge calls. */
     expectedReps?: number;
+    /** Select only repetitions with retained ERROR criterion votes. */
+    onlyUnparsed?: boolean;
+    prior?: ScenarioResult;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -72,6 +75,8 @@ export interface RegradeRunOptions {
      * cleanly, so spending judge calls on them buys nothing.
      */
     onlySuspect?: boolean;
+    /** Rejudge only retained repetitions containing ERROR criterion votes, with the recorded judge. */
+    onlyUnparsed?: boolean;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own

package/dist/regrade.js

@@ -56,18 +56,18 @@ export async function judgeOneRep(opts) {
         };
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
-    const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
+    const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
     appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
     if (g.suspect)
         appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
     return {
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
-            judge_calls: 1,
-            judge_rejudge_calls: opts.rejudge ? 1 : 0,
+            judge_calls: 1 + (g.judgeRetries ?? 0),
+            judge_rejudge_calls: opts.rejudge ? 1 + (g.judgeRetries ?? 0) : 0,
         },
     };
 }
@@ -81,18 +81,30 @@ export async function judgeOneRep(opts) {
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
     const mode = opts.mode ?? "green";
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    let files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    if (opts.onlyUnparsed)
+        files = files.filter(file => hasUnparsedVotes(opts.prior?.rep_judgments?.find(panel => panel.repetition === (repIndexOf(file) ?? 0))));
     if (files.length === 0)
         throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const expected = opts.expectedReps ?? files.length;
-    const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
+    const expectedIndices = opts.onlyUnparsed
+        ? opts.prior.rep_judgments.filter(hasUnparsedVotes).map(panel => expected === 1 ? null : panel.repetition).sort((a, b) => (a ?? -1) - (b ?? -1))
+        : expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
     const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
-    if (files.length !== expected || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
+    if (files.length !== expectedIndices.length || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
         throw new Error(`${opts.scenario.id}: transcript artifacts are incomplete for ${expected} recorded rep(s) — re-run instead of grading a smaller repetition set`);
     }
     const repCount = expected;
     const outcomes = [];
-    for (const file of files) {
+    for (let repetition = 0; repetition < expected; repetition++) {
+        const panel = opts.prior?.rep_judgments?.find(panel => panel.repetition === repetition);
+        if (opts.onlyUnparsed && !hasUnparsedVotes(panel)) {
+            if (!panel)
+                throw new Error(`missing retained judgments for ${opts.scenario.id}#${repetition}`);
+            outcomes.push({ verdict: panel.recorded_verdict, reason: panel.judgments[0]?.reason ?? "", suspect: panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")), judgment: panel.judgments[0], objective: panel.objective });
+            continue;
+        }
+        const file = files.find(file => (repIndexOf(file) ?? 0) === repetition);
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
@@ -100,7 +112,16 @@ export async function regradeScenario(opts) {
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
         }));
     }
-    return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    if (opts.onlyUnparsed)
+        result.rep_judgments = result.rep_judgments?.map(panel => {
+            const prior = opts.prior.rep_judgments.find(prior => prior.repetition === panel.repetition);
+            return hasUnparsedVotes(prior) ? panel : prior;
+        });
+    return result;
+}
+function hasUnparsedVotes(panel) {
+    return panel?.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? false;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own
@@ -114,17 +135,21 @@ export async function regradeScenario(opts) {
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
 export async function regradeRun(opts) {
-    const { runDir, spec, adapter, judge, specDir } = opts;
+    const { runDir, spec, adapter, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
+    if (opts.onlyUnparsed && !prev)
+        throw new Error(`--unparsed-only needs a prior results.yaml in ${runDir}`);
+    if (opts.onlyUnparsed && opts.onlySuspect)
+        throw new Error("--unparsed-only cannot be combined with --suspect-only");
+    const judge = opts.onlyUnparsed ? prev.judge : opts.judge;
     // Carried across a re-judge, per field, because the right answer differs:
     //  - `override`/`note`  — the author's, never the judge's to discard.
     //  - `objective`        — a re-judge does NOT re-evaluate trace gates (that is
     //    `regate`), so the recorded evidence still describes this run. Dropping it
     //    silently downgraded a gated scenario to "no assertions declared".
-    // `adjudication` is deliberately NOT carried: it describes the judgments this
-    // re-grade just replaced, and a stale panel beside a fresh verdict is worse
-    // than none.
+    // `adjudication` is dropped when its repetition is rejudged. A selective
+    // repair of a different repetition carries that untouched evidence instead.
     // The whole prior result per id — `rebuildScenarioResult` decides, field by
     // field, what survives. Passing a hand-picked subset here is how fields got
     // dropped before.
@@ -148,6 +173,8 @@ export async function regradeRun(opts) {
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
     }
+    if (opts.onlyUnparsed)
+        targets = recorded.filter(s => s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
     // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
     // cannot turn an undelivered or unauthenticated transcript into product evidence.
     if (prev?.schema === 3) {
@@ -161,6 +188,9 @@ export async function regradeRun(opts) {
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
         const files = findTranscriptFiles(runDir, record.id, mode);
+        if (opts.onlyUnparsed) {
+            return record.rep_judgments?.length === expected && record.rep_judgments.every(panel => !hasUnparsedVotes(panel) || files.some(file => (repIndexOf(file) ?? 0) === panel.repetition)) || false;
+        }
         const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
         const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
         return files.length === expected && JSON.stringify(actualIndices) === JSON.stringify(expectedIndices);
@@ -173,6 +203,14 @@ export async function regradeRun(opts) {
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
     }
+    if (opts.onlyUnparsed) {
+        const drifted = targets.filter(id => {
+            const count = recordedById.get(id).criterion_count;
+            return count !== undefined && count !== specById.get(id).checklist.length;
+        });
+        if (drifted.length)
+            throw new Error(`criterion count changed for ${drifted.join(", ")} — use full grade instead of --unparsed-only`);
+    }
     const targetSet = new Set(targets);
     const scenarioResults = [];
     for (const rec of recorded) {
@@ -186,17 +224,17 @@ export async function regradeRun(opts) {
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
             runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
-            expectedReps: prevScenario?.reps ?? 1,
+            expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
         });
         const carry = overrides.get(id);
         if (prev?.schema === 3)
             rr.criterion_count = scenario.checklist.length;
         rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
         rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
-        // `grade` re-judges the saved transcript. It does not re-evaluate trace gates
-        // (that is `regate`), so `objective` still describes this run; and it replaced
-        // the judgments a prior adjudication described, so that panel must go.
-        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: "drop" }));
+        // Gates still describe the original subject execution. An adjudication of
+        // an untouched repetition also survives, bounded by rebuild's conflict guards.
+        const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
+        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
     }
     const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
@@ -229,7 +267,7 @@ export async function regradeRun(opts) {
         // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
         // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
-        source_hashes: refreshRubricHashes(prev?.source_hashes, spec, targets),
+        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets),
         source_hash_roots: prev?.source_hash_roots,
         scenarios: scenarioResults,
     }, ctx);

package/dist/reps.js

@@ -16,9 +16,11 @@ export function aggregateObjective(outcomes) {
     if (present.length === 1)
         return picked;
     const traceHashes = present.map((objective) => objective.trace_sha256);
+    const outputHashes = present.map((objective) => objective.output_sha256);
     return {
         ...picked,
         ...(traceHashes.every((hash) => typeof hash === "string") ? { rep_trace_sha256: traceHashes } : {}),
+        ...(outputHashes.every((hash) => typeof hash === "string") ? { rep_output_sha256: outputHashes } : {}),
     };
 }
 /**

package/dist/results.d.ts

@@ -99,6 +99,8 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
+    /** One retry with identical judge input after unparseable criterion votes. */
+    judgeRetries?: 1;
 }
 export interface AdjudicationResult {
     /** The saved transcript repetition judged by every panel member (first rep by policy). */
@@ -117,8 +119,11 @@ export interface ObjectiveResult {
     status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
     trace_version?: number;
     trace_sha256?: string;
+    /** SHA-256 of the final assistant message evaluated by output regex gates. */
+    output_sha256?: string;
     /** Per-repetition hashes retained when an aggregate has more than one rep. */
     rep_trace_sha256?: string[];
+    rep_output_sha256?: string[];
     assertions: {
         kind: string;
         status: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";

package/dist/results.js

@@ -613,7 +613,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
     const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
-        (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+        (pickedAdjudication.verdict === "PASS" && pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
     const adjudication = conflictsWithFreshEvidence && pickedAdjudication

package/dist/run.js

@@ -6,7 +6,7 @@ import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
-import { runSeeded } from "./seeded.js";
+import { evaluateOutputGates, hasOutputGates, runSeeded } from "./seeded.js";
 import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
@@ -194,11 +194,11 @@ export async function runSkillModel(opts) {
  * ends the last assistant section.
  */
 export function hasEmptyAssistantTurn(transcript) {
-    const sections = transcript.split(/^<<< ASSISTANT:\s*$/m).slice(1);
+    const sections = transcript.split(/^<<< ASSISTANT:[ \t]*$/m).slice(1);
     if (sections.length === 0)
         return false;
     return sections.some((sec) => {
-        const body = sec.split(/^(?:>>> |=== SEEDED GATES ===|\[pi exited )/m)[0];
+        const body = sec.split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*|\[pi exited [^\]]+\][ \t]*)$/m)[0];
         return body.trim() === "";
     });
 }
@@ -396,6 +396,9 @@ async function runRep(scenario, rep, repCount, ctx) {
             }
         }
         const repSuffix = repCount > 1 ? rep : undefined;
+        const outputGate = !adapterFailure && !noResponse && !infrastructureFailure
+            ? evaluateOutputGates(scenario, transcript)
+            : { status: "PASS", failure: null, assertions: [], lines: [] };
         writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
         if (scenario.mode === "seeded") {
             // The workspace is torn down in the `finally` below, so this is the only
@@ -440,10 +443,11 @@ async function runRep(scenario, rep, repCount, ctx) {
         // timeout that caused it, and it is that text, not the real reason, that would
         // reach the results record. The verdict is ERROR either way.
         let objective;
-        if (scenario.traceAssert && !adapterFailure) {
-            const assertionResults = [];
-            let status = "PASS";
+        if ((scenario.traceAssert || hasOutputGates(scenario)) && !adapterFailure && !noResponse && !infrastructureFailure) {
+            const assertionResults = [...outputGate.assertions];
+            let status = outputGate.status;
             let traceMeta = {};
+            const outputMeta = outputGate.outputSha256 ? { output_sha256: outputGate.outputSha256 } : {};
             if (scenario.traceAssert) {
                 if (traces.length > 0) {
                     writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
@@ -459,12 +463,15 @@ async function runRep(scenario, rep, repCount, ctx) {
                 }
                 else {
                     const gate = evaluateTraceGates(scenario.traceAssert, merged);
-                    status = gate.status;
+                    if (gate.status === "ERROR" || status === "ERROR")
+                        status = "ERROR";
+                    else if (gate.status === "FAIL" || status === "FAIL")
+                        status = "FAIL";
                     assertionResults.push(...gate.assertions);
                     traceMeta = { trace_version: merged.trace_version, trace_sha256: merged.trace_sha256 };
                 }
             }
-            objective = { status, ...traceMeta, assertions: assertionResults };
+            objective = { status, ...traceMeta, ...outputMeta, assertions: assertionResults };
             if (status !== "PASS") {
                 const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
                 gatePrefix = `objective: ${details.join("; ") || "structured evidence could not be evaluated"}`;
@@ -514,7 +521,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             reason = o.reason;
             suspect = o.suspect;
             judgment = o.judgment; // judgeOneRep already journaled (verdict + misfire)
-            judgeCalls = 1;
+            judgeCalls = o.metrics?.judge_calls ?? 0;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
         const subject = mergeTraces(traces)?.metrics;

package/dist/seeded.d.ts

@@ -82,6 +82,28 @@ export interface SeededOutcome {
 export declare function changedLines(diff: string): string;
 /** Whether a scenario declares any needle gate at all. */
 export declare function hasNeedleGates(scenario: Scenario): boolean;
+/** Whether a scenario declares any final-assistant-message gate. */
+export declare function hasOutputGates(scenario: Scenario): boolean;
+/**
+ * The final assistant turn exactly as the transcript artifact records its text.
+ *
+ * Delimiter-shaped model output is ambiguous in this plain-text artifact. Refuse
+ * it instead of truncating the output and turning a forbidden suffix into PASS.
+ */
+export declare function finalAssistantMessage(transcript: string, scenario: Scenario): string | null;
+export interface OutputGateResult {
+    status: "PASS" | "FAIL" | "ERROR";
+    failure: string | null;
+    outputSha256?: string;
+    assertions: {
+        kind: "output_matches" | "output_excludes" | "output_evidence";
+        status: "PASS" | "FAIL" | "ERROR";
+        detail: string;
+    }[];
+    lines: string[];
+}
+/** Evaluate final-message regex gates without exposing the matched text. */
+export declare function evaluateOutputGates(scenario: Scenario, transcript: string): OutputGateResult;
 /**
  * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
  * to report, and the first failure (null when every needle is satisfied).

package/dist/seeded.js

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { copyFileSync, statSync } from "node:fs";
 import { extname, isAbsolute, join, resolve } from "node:path";
 import { exec } from "./util/exec.js";
@@ -60,6 +61,74 @@ export function changedLines(diff) {
 export function hasNeedleGates(scenario) {
     return (scenario.assert?.diff_contains?.length ?? 0) > 0 || (scenario.assert?.diff_excludes?.length ?? 0) > 0;
 }
+/** Whether a scenario declares any final-assistant-message gate. */
+export function hasOutputGates(scenario) {
+    return (scenario.assert?.output_matches?.length ?? 0) > 0 || (scenario.assert?.output_excludes?.length ?? 0) > 0;
+}
+/**
+ * The final assistant turn exactly as the transcript artifact records its text.
+ *
+ * Delimiter-shaped model output is ambiguous in this plain-text artifact. Refuse
+ * it instead of truncating the output and turning a forbidden suffix into PASS.
+ */
+export function finalAssistantMessage(transcript, scenario) {
+    const assistantMarker = /^<<< ASSISTANT:[ \t]*$/gm;
+    const userMarker = /^>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*$/gm;
+    const gateMarker = /^=== SEEDED GATES ===[ \t]*$/gm;
+    const exitMarker = /^\[pi exited [^\]]+\][ \t]*$/gm;
+    const assistantCount = [...transcript.matchAll(assistantMarker)].length;
+    const userCount = [...transcript.matchAll(userMarker)].length;
+    const gateCount = [...transcript.matchAll(gateMarker)].length;
+    const exitCount = [...transcript.matchAll(exitMarker)].length;
+    const expectedGates = scenario.mode === "seeded" ? 1 : 0;
+    if (assistantCount !== scenario.turns.length || userCount !== scenario.turns.length || gateCount !== expectedGates || exitCount !== 0) {
+        return null;
+    }
+    const sections = transcript.split(/^<<< ASSISTANT:[ \t]*$/m);
+    return sections[sections.length - 1]
+        .split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*|\[pi exited [^\]]+\][ \t]*)$/m)[0]
+        .trim();
+}
+/** Evaluate final-message regex gates without exposing the matched text. */
+export function evaluateOutputGates(scenario, transcript) {
+    if (!hasOutputGates(scenario))
+        return { status: "PASS", failure: null, assertions: [], lines: [] };
+    const output = finalAssistantMessage(transcript, scenario);
+    if (output === null) {
+        const detail = "final assistant message is missing or transcript delimiters are ambiguous";
+        return {
+            status: "ERROR", failure: `output evidence: ${detail}`,
+            assertions: [{ kind: "output_evidence", status: "ERROR", detail }],
+            lines: [`  output: ERROR (${detail})`],
+        };
+    }
+    const assertions = [];
+    const lines = [];
+    let failure = null;
+    for (const pattern of scenario.assert?.output_matches ?? []) {
+        const ok = new RegExp(pattern, "m").test(output);
+        const detail = ok ? `final assistant message matches ${JSON.stringify(pattern)}` : `final assistant message does not match ${JSON.stringify(pattern)}`;
+        assertions.push({ kind: "output_matches", status: ok ? "PASS" : "FAIL", detail });
+        lines.push(`  output_matches ${JSON.stringify(pattern)}: ${ok ? "MATCH" : "MISSING"}`);
+        if (!ok && !failure)
+            failure = `output_matches ${JSON.stringify(pattern)} did not match the final assistant message`;
+    }
+    for (const pattern of scenario.assert?.output_excludes ?? []) {
+        const ok = !new RegExp(pattern, "m").test(output);
+        const detail = ok ? `final assistant message excludes ${JSON.stringify(pattern)}` : `final assistant message matches forbidden regex ${JSON.stringify(pattern)}`;
+        assertions.push({ kind: "output_excludes", status: ok ? "PASS" : "FAIL", detail });
+        lines.push(`  output_excludes ${JSON.stringify(pattern)}: ${ok ? "ABSENT" : "PRESENT"}`);
+        if (!ok && !failure)
+            failure = `output_excludes ${JSON.stringify(pattern)} matched the final assistant message`;
+    }
+    return {
+        status: failure ? "FAIL" : "PASS",
+        failure,
+        outputSha256: createHash("sha256").update(output).digest("hex"),
+        assertions,
+        lines,
+    };
+}
 /**
  * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
  * to report, and the first failure (null when every needle is satisfied).

package/dist/sources.d.ts

@@ -54,7 +54,7 @@ export declare const FIXTURE_PREFIX = "fixture:";
  * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
  * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
  * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
- * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
+ * | `gates:<id>` | diff/output regex gates and `assert.trace` | gate wrong, saved evidence still valid | `regate` (no subject call; judges fail→pass reps) |
  * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
  *
  * Why this matters more than it looks: with one key, lint had exactly one remedy for
@@ -105,7 +105,7 @@ export declare function dirSha256(dir: string): string | null;
 export declare function stimulusDigest(s: Scenario): string;
 export declare function rubricDigest(s: Scenario): string;
 export declare function policyDigest(s: Scenario): string;
-/** Null when the scenario declares no needle gates — no key is recorded for it. */
+/** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export declare function gatesDigest(s: Scenario): string | null;
 /** The spec-level judge persona, which is rubric for every scenario at once. */
 export declare function personaDigest(persona: string): string;

package/dist/sources.js

@@ -58,7 +58,7 @@ export const FIXTURE_PREFIX = "fixture:";
  * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
  * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
  * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
- * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
+ * | `gates:<id>` | diff/output regex gates and `assert.trace` | gate wrong, saved evidence still valid | `regate` (no subject call; judges fail→pass reps) |
  * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
  *
  * Why this matters more than it looks: with one key, lint had exactly one remedy for
@@ -163,7 +163,7 @@ function walk(dir, prefix = "") {
  * A scenario's fields, sorted into the four buckets, as canonical JSON.
  *
  * One function so the exhaustive-destructure trick below covers all four digests at
- * once: adding a field to `Scenario` or `SeededAssert` fails the build **here** until
+ * once: adding a field to `Scenario` or `ScenarioAssert` fails the build **here** until
  * someone decides which bucket — and therefore which remedy — it belongs to. A field
  * nobody assigned is a permanent staleness blind spot, and a field assigned to the
  * wrong bucket is worse than that: it would tell a user `rescore` is enough when the
@@ -180,14 +180,14 @@ function facets(s) {
     // anywhere would charge a re-run (or at best a re-score) for editing a label,
     // which is the exact trap the facet split was built to remove.
     void _coversIsMetadata;
-    const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
+    const { vitest, diff_contains, diff_excludes, output_matches, output_excludes, post_test, ...restAssert } = assert ?? {};
     const _assertExhaustive = restAssert;
     void _assertExhaustive;
     // `traceAssert` is a GATE, not stimulus: it is evaluated against a trace the run
     // already saved, so `regate` can re-answer it without re-running the subject model.
     // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
     // changes what gets executed, the other only what we conclude from it.
-    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || output_matches !== undefined || output_excludes !== undefined || traceAssert !== undefined;
     return {
         // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
         // run executes in the workspace, and neither can be re-evaluated from a saved
@@ -214,6 +214,8 @@ function facets(s) {
         gates: hasGates
             ? JSON.stringify([
                 id, diff_contains ?? null, diff_excludes ?? null,
+                ...(output_matches ? [["output_matches", output_matches]] : []),
+                ...(output_excludes ? [["output_excludes", output_excludes]] : []),
                 ...(traceAssert ? [traceAssert] : []),
             ])
             : null,
@@ -231,7 +233,7 @@ export function rubricDigest(s) {
 export function policyDigest(s) {
     return sha(facets(s).policy);
 }
-/** Null when the scenario declares no needle gates — no key is recorded for it. */
+/** Null when the scenario declares no offline-replayable gates — no key is recorded for it. */
 export function gatesDigest(s) {
     const g = facets(s).gates;
     return g === null ? null : sha(g);

package/dist/spec.d.ts

@@ -1,7 +1,7 @@
 import type { WorkspaceKind } from "./workspace.js";
 import { type TraceAssert } from "./trace-gates.js";
 export type ScenarioMode = "inline" | "seeded";
-export interface SeededAssert {
+export interface ScenarioAssert {
     vitest?: boolean;
     diff_contains?: string[];
     /**
@@ -11,6 +11,10 @@ export interface SeededAssert {
      * from the model's prose.
      */
     diff_excludes?: string[];
+    /** Regular expressions that must all match the final assistant message. */
+    output_matches?: string[];
+    /** Regular expressions that must not match the final assistant message. */
+    output_excludes?: string[];
     /**
      * A test file copied into the workspace AFTER the agent finishes, then run.
      * The model never sees it, so it cannot write code shaped to pass it — this
@@ -28,12 +32,12 @@ export interface Scenario {
     turns: string[];
     checklist: string[];
     fixture?: string;
-    assert?: SeededAssert;
+    assert?: ScenarioAssert;
     /**
      * Objective assertions over the execution trace.
      *
-     * Deliberately NOT part of `SeededAssert`: the other gates read a staged git
-     * diff and are meaningless without a fixture, while a trace exists for any run.
+     * Deliberately NOT part of `ScenarioAssert`: trace assertions have their own
+     * closed schema and opt execution into structured capture.
      * Declaring it opts the scenario into structured (`--mode json`) execution.
      */
     traceAssert?: TraceAssert;

package/dist/spec.js

@@ -11,6 +11,21 @@ export class SpecError extends Error {
 function isStringArray(v) {
     return Array.isArray(v) && v.every((x) => typeof x === "string");
 }
+function parseRegexList(v, id, field, file) {
+    assertStringList(v, id, field, file);
+    for (const pattern of v) {
+        if (pattern === "") {
+            throw new SpecError(`scenario \`${id}\` \`${field}\` patterns must not be empty`, file);
+        }
+        try {
+            new RegExp(pattern, "m");
+        }
+        catch (e) {
+            throw new SpecError(`scenario \`${id}\` \`${field}\` contains invalid regular expression ${JSON.stringify(pattern)} — ${e.message}`, file);
+        }
+    }
+    return v;
+}
 /**
  * Require a non-empty list of strings, with a targeted error. A common authoring
  * trap: an unquoted "key: value" list item parses as a YAML mapping, not a string
@@ -176,23 +191,39 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
-        // `assert.trace` is legal for inline AND seeded scenarios — it reads the
-        // execution trace, which every run produces, not a staged diff.
+        // Output and trace assertions are legal for inline AND seeded scenarios.
+        if (s.assert !== undefined && (s.assert === null || typeof s.assert !== "object" || Array.isArray(s.assert))) {
+            throw new SpecError(`scenario \`${id}\` \`assert\` must be a mapping`, file);
+        }
         const rawAssert = s.assert;
         if (rawAssert?.trajectory !== undefined) {
             throw new SpecError(`scenario \`${id}\` uses removed \`assert.trajectory\`; delete it or replace it with an active objective gate`, file);
         }
+        const allowedAssertKeys = new Set([
+            "vitest", "diff_contains", "diff_excludes", "post_test", "trace", "output_matches", "output_excludes",
+        ]);
+        for (const key of Object.keys(rawAssert ?? {})) {
+            if (!allowedAssertKeys.has(key)) {
+                throw new SpecError(`scenario \`${id}\` has unknown \`assert\` key \`${key}\``, file);
+            }
+        }
         if (rawAssert?.trace !== undefined) {
             scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
         }
+        const assertObj = {};
+        if (rawAssert?.output_matches !== undefined) {
+            assertObj.output_matches = parseRegexList(rawAssert.output_matches, id, "assert.output_matches", file);
+        }
+        if (rawAssert?.output_excludes !== undefined) {
+            assertObj.output_excludes = parseRegexList(rawAssert.output_excludes, id, "assert.output_excludes", file);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
             }
             scenario.fixture = s.fixture;
-            const a = s.assert;
+            const a = rawAssert;
             if (a) {
-                const assertObj = {};
                 if (a.vitest !== undefined)
                     assertObj.vitest = a.vitest === true;
                 if (a.diff_contains !== undefined) {
@@ -232,9 +263,10 @@ export function parseSpec(text, file) {
                     }
                     assertObj.post_test = a.post_test.trim();
                 }
-                scenario.assert = assertObj;
             }
         }
+        if (Object.keys(assertObj).length > 0)
+            scenario.assert = assertObj;
         if (s.env && typeof s.env === "object" && Object.hasOwn(s.env, "event_sources")) {
             throw new SpecError(`scenario \`${id}\` uses removed \`env.event_sources\`; it is no longer collected`, file);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.21.0",
+    "version": "0.22.3",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -232,6 +232,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,
@@ -353,6 +354,10 @@
             "trace_sha256": {
               "type": "string"
             },
+            "output_sha256": {
+              "type": "string",
+              "pattern": "^[a-fA-F0-9]{64}$"
+            },
             "rep_trace_sha256": {
               "type": "array",
               "items": {
@@ -360,6 +365,13 @@
                 "pattern": "^[a-fA-F0-9]{64}$"
               }
             },
+            "rep_output_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
+            },
             "assertions": {
               "type": "array",
               "items": {

package/schemas/results-v3.schema.json

@@ -244,6 +244,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,
@@ -380,6 +381,10 @@
             "trace_sha256": {
               "type": "string"
             },
+            "output_sha256": {
+              "type": "string",
+              "pattern": "^[a-fA-F0-9]{64}$"
+            },
             "trajectory_version": {
               "const": "1.0"
             },
@@ -400,6 +405,13 @@
                 "pattern": "^[a-fA-F0-9]{64}$"
               }
             },
+            "rep_output_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
+            },
             "assertions": {
               "type": "array",
               "items": {

package/schemas/specification-v1.schema.json

@@ -157,6 +157,22 @@
                 "minLength": 1
               }
             },
+            "output_matches": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "output_excludes": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
             "post_test": {
               "type": "string"
             },