@skill-harness/core 0.22.3 → 0.23.0

package/dist/grade.d.ts

@@ -36,13 +36,23 @@ export declare function parseVerdict(out: string): ParsedVerdict;
  */
 export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
 export interface GradeResult extends ParsedVerdict {
+    /** The final judge reply, retained for callers that predate structured judging. */
     raw: string;
+    /** Every attempt in call order; two entries mean the structured retry also ran. */
+    rawReplies: string[];
     criteria: CriterionVote[];
-    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeFormat: "json";
+    /** One validation-guided retry after an invalid structured reply. Absent on the first attempt. */
     judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export declare function parseStructuredJudgeReply(raw: string, expectedCriteria: number): ParsedVerdict & {
+    criteria: CriterionVote[];
+};
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export declare function formatJudgeRawReplies(replies: string[]): string;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
  * the overall verdict equals AND(items). A mismatch in EITHER direction — verdict
@@ -51,11 +61,11 @@ export interface GradeResult extends ParsedVerdict {
  * parse, or the verdict is ERROR, return false (never block a run on a parse miss).
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
-/** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
+/** Drive the judge for one transcript and parse the JSON-only result. */
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
+import { completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -37,9 +37,8 @@ ${numbered}
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
 ${diffGuidance}
-Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
-VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
-REASON: <15 words or fewer>`;
+Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Reply with exactly one JSON object and nothing else, with one vote per criterion in checklist order:
+{"votes":[{"criterion":1,"vote":"PASS"|"FAIL","reason":"..."},...],"verdict":"PASS"|"FAIL","reason":"..."}`;
 }
 // Both anchor to the start of a line and REQUIRE the colon. Without those anchors the
 // reason pattern matched any word containing "reason" — git-ops GLM C1 stored
@@ -87,6 +86,85 @@ export function judgeResemblesSubject(judge, subject) {
     const b = subject.model;
     return a === b || a.includes(b) || b.includes(a);
 }
+/** Extract the first complete top-level JSON object, tolerating prose or fences around it. */
+function extractFirstJsonObject(raw) {
+    const start = raw.indexOf("{");
+    if (start < 0)
+        throw new Error("no JSON object in judge reply");
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === "\\")
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"')
+            inString = true;
+        else if (ch === "{")
+            depth++;
+        else if (ch === "}" && --depth === 0)
+            return raw.slice(start, i + 1);
+    }
+    throw new Error("incomplete JSON object in judge reply");
+}
+function exactKeys(value, expected, context) {
+    const actual = Object.keys(value).sort();
+    const wanted = [...expected].sort();
+    if (JSON.stringify(actual) !== JSON.stringify(wanted)) {
+        throw new Error(`${context} must contain exactly ${wanted.join(", ")}`);
+    }
+}
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export function parseStructuredJudgeReply(raw, expectedCriteria) {
+    let value;
+    try {
+        value = JSON.parse(extractFirstJsonObject(raw));
+    }
+    catch (error) {
+        if (error instanceof Error && /JSON object in judge reply/.test(error.message))
+            throw error;
+        throw new Error(`invalid JSON: ${error.message}`);
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error("judge reply must be a JSON object");
+    const object = value;
+    exactKeys(object, ["votes", "verdict", "reason"], "judge reply");
+    if (!Array.isArray(object.votes))
+        throw new Error("votes must be an array");
+    if (object.votes.length !== expectedCriteria)
+        throw new Error(`expected ${expectedCriteria} votes, got ${object.votes.length}`);
+    const criteria = object.votes.map((rawVote, offset) => {
+        if (!rawVote || typeof rawVote !== "object" || Array.isArray(rawVote))
+            throw new Error(`vote ${offset + 1} must be an object`);
+        const vote = rawVote;
+        exactKeys(vote, ["criterion", "vote", "reason"], `vote ${offset + 1}`);
+        if (vote.criterion !== offset + 1)
+            throw new Error(`vote ${offset + 1} must have criterion ${offset + 1}`);
+        if (vote.vote !== "PASS" && vote.vote !== "FAIL")
+            throw new Error(`vote ${offset + 1} must be PASS or FAIL`);
+        if (typeof vote.reason !== "string")
+            throw new Error(`vote ${offset + 1} reason must be a string`);
+        return { index: offset + 1, verdict: vote.vote, reason: vote.reason };
+    });
+    if (object.verdict !== "PASS" && object.verdict !== "FAIL")
+        throw new Error("verdict must be PASS or FAIL");
+    if (typeof object.reason !== "string")
+        throw new Error("reason must be a string");
+    return { criteria, verdict: object.verdict, reason: object.reason };
+}
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export function formatJudgeRawReplies(replies) {
+    if (replies.length === 1)
+        return replies[0];
+    return replies.map((reply, index) => `=== JUDGE REPLY ${index + 1} ===\n${reply}`).join("\n\n");
+}
 const ITEM_RE = /^\s*\d+[.)]\s*\**\s*(PASS|FAIL)\b/gim;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
@@ -122,27 +200,48 @@ export function detectMisfire(raw, verdict) {
 }
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
-/** Drive the judge for one transcript and parse the result. */
+/** Drive the judge for one transcript and parse the JSON-only result. */
 export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
-    const request = { model: judge, prompt, cwd };
-    let raw = await adapter.judge(request);
-    let criteria = parseCriterionVotes(raw);
-    let judgeRetries;
-    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
-        raw = await adapter.judge(request);
-        criteria = parseCriterionVotes(raw);
-        judgeRetries = 1;
-    }
-    const parsed = parseVerdict(raw);
-    // On a parse failure, surface what the judge actually emitted (e.g. a provider
-    // error) rather than a generic message — otherwise the cause is invisible.
-    if (parsed.verdict === "ERROR") {
-        const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
-        if (snippet)
-            parsed.reason = `judge unparseable: ${snippet}`;
+    const rawReplies = [];
+    let validationError = "";
+    for (let attempt = 0; attempt < 2; attempt++) {
+        const retry = attempt === 0 ? "" : `\n\nYour reply was not valid: ${JSON.stringify(validationError)}; reply with only the JSON object.`;
+        const raw = await adapter.judge({ model: judge, prompt: prompt + retry, cwd });
+        rawReplies.push(raw);
+        // Pi adapter failures are infrastructure errors, not malformed model votes.
+        if (/^\[judge error:/i.test(raw.trim())) {
+            const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
+            return {
+                verdict: "ERROR", reason: `judge unparseable: ${snippet}`, suspect: false,
+                raw, rawReplies, criteria: completeCriterionVotes([], expectedCriteria), judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        try {
+            const parsed = parseStructuredJudgeReply(raw, expectedCriteria);
+            const suspect = parsed.verdict === "FAIL"
+                ? parsed.criteria.every(vote => vote.verdict === "PASS")
+                : parsed.criteria.some(vote => vote.verdict === "FAIL");
+            return {
+                ...parsed, raw, rawReplies, suspect, judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        catch (error) {
+            validationError = error.message;
+        }
     }
-    const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
+    const raw = rawReplies[rawReplies.length - 1];
+    return {
+        verdict: "UNGRADED",
+        reason: `judge structured reply invalid after retry: ${validationError}`,
+        suspect: false,
+        raw,
+        rawReplies,
+        criteria: completeCriterionVotes([], expectedCriteria),
+        judgeFormat: "json",
+        judgeRetries: 1,
+    };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the

package/dist/lift.js

@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
     // misfire are judge failures. Treating any of them as a FAIL would make
     // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
     // measure flakiness while looking like skill value.
-    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
+    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS" && verdict !== "UNGRADED";
 }
 function classify(red, green) {
     if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))

package/dist/regate.js

@@ -49,11 +49,19 @@ function rewriteTranscript(path, gateLines) {
     renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
     writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
 }
-/** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
-function verdictFromSavedJudgement(runDir, id, mode, rep) {
+/** Recover a rep's judge verdict from retained structured evidence or its legacy prose artifact. */
+function verdictFromSavedJudgement(runDir, id, mode, rep, retained) {
     const path = judgeRawPath(runDir, id, mode, rep);
     if (!existsSync(path))
         return null;
+    if (retained?.judgeFormat === "json") {
+        return {
+            verdict: retained.verdict,
+            reason: retained.reason,
+            suspect: retained.suspect,
+            judgment: retained,
+        };
+    }
     const raw = readFileSync(path, "utf8");
     const parsed = parseVerdict(raw);
     return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
@@ -284,7 +292,9 @@ export async function regateRun(opts) {
             if (!oldGateFailed) {
                 // The judge already saw this rep. Its verdict is on disk — re-read it rather
                 // than paying to ask the same question again.
-                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
+                const panel = rec.rep_judgments?.find(candidate => candidate.repetition === (rep ?? 0));
+                const retained = panel?.judgments.find(judgment => judgment.ordinal === 1) ?? panel?.judgments[0];
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep, retained);
                 outcomes.push({ ...(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect }), objective });
                 continue;
             }

package/dist/regrade.d.ts

@@ -53,6 +53,8 @@ export declare function judgeOneRep(opts: {
     now: () => string;
     /** True when this call revisits a saved subject transcript. */
     rejudge?: boolean;
+    /** Retained objective evidence must precede missing-vote normalization. */
+    objective?: RepOutcome["objective"];
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no

package/dist/regrade.js

@@ -1,8 +1,8 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
+import { buildJudgePrompt, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
 import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
-import { outcomesToResult } from "./reps.js";
+import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript } from "./provider-failure.js";
@@ -57,19 +57,20 @@ export async function judgeOneRep(opts) {
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
-    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
-    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
-    if (g.suspect)
-        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
-    return {
-        verdict: g.verdict, reason: g.reason, suspect: g.suspect,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
+    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), formatJudgeRawReplies(g.rawReplies), "utf8");
+    const outcome = normalizeRepOutcome({
+        verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),
             judge_rejudge_calls: opts.rejudge ? 1 + (g.judgeRetries ?? 0) : 0,
         },
-    };
+    });
+    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: outcome.verdict, reason: outcome.reason, suspect: outcome.suspect, ...repField });
+    if (outcome.suspect)
+        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: outcome.reason, ...repField });
+    return outcome;
 }
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -110,6 +111,7 @@ export async function regradeScenario(opts) {
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
+            objective: panel?.objective ?? (expected === 1 ? opts.prior?.objective : undefined),
         }));
     }
     const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);

package/dist/report.d.ts

@@ -26,6 +26,7 @@ export interface RunColumn {
         passes?: number;
         clean?: number;
         flakiness?: number;
+        ungraded_reps?: number;
         metrics?: import("./results.js").ScenarioMetrics;
         override: string | null;
         note: string;
@@ -109,6 +110,7 @@ export declare function publicView(data: ReportData): {
             passes?: number;
             clean?: number;
             flakiness?: number;
+            ungraded_reps?: number;
             metrics?: import("./results.js").ScenarioMetrics;
             override: string | null;
             note: string;

package/dist/report.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults } from "./results.js";
+import { readResults, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
 import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
@@ -40,9 +40,13 @@ export function collectReport(skillDir) {
             if (!runDir)
                 continue;
             const r = readResults(runDir);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
             const tagName = tagDir.split("/").pop();
             const cells = {};
-            for (const s of r.scenarios) {
+            for (const s of normalized) {
                 const boundary = boundaryByCell.get(`${tagName}\u0000${r.mode}\u0000${s.id}`);
                 cells[s.id] = {
                     ...(boundary
@@ -72,6 +76,7 @@ export function collectReport(skillDir) {
                     passes: s.passes,
                     clean: s.clean,
                     flakiness: s.flakiness,
+                    ...(s.ungraded_reps ? { ungraded_reps: s.ungraded_reps } : {}),
                     metrics: s.metrics,
                     override: s.override,
                     note: s.note,
@@ -96,7 +101,7 @@ export function collectReport(skillDir) {
                 timestamp: r.timestamp,
                 mode: r.mode,
                 partial: r.partial === true,
-                grade: r.effective_grade,
+                grade,
                 judge: r.judge,
                 metrics: aggregateMetrics(r.scenarios),
                 cells,

package/dist/reps.d.ts

@@ -16,6 +16,8 @@ export interface RepOutcome {
         subject?: TraceMetrics;
     };
 }
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export declare function normalizeRepOutcome(outcome: RepOutcome): RepOutcome;
 /**
  * Collapse per-rep objective results.
  *

package/dist/reps.js

@@ -1,3 +1,17 @@
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export function normalizeRepOutcome(outcome) {
+    if (outcome.verdict === "ERROR")
+        return outcome;
+    if (outcome.objective && outcome.objective.status !== "PASS") {
+        return { ...outcome, verdict: outcome.objective.status, suspect: false };
+    }
+    if (outcome.verdict === "NOT-MEASURED")
+        return outcome;
+    if (outcome.verdict === "UNGRADED" || outcome.judgment?.criteria?.some(vote => vote.verdict === "ERROR")) {
+        return { ...outcome, verdict: "UNGRADED", reason: "UNGRADED: incomplete criterion votes after judge retry", suspect: false };
+    }
+    return outcome;
+}
 /**
  * Collapse per-rep objective results.
  *
@@ -31,7 +45,9 @@ export function aggregateObjective(outcomes) {
  * (default caller threshold 0.5, ties pass). Flakiness = 1 - |2·pass_rate - 1|.
  */
 export function aggregateReps(outcomes, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
     const reps = outcomes.length;
+    const ungraded = outcomes.filter(o => o.verdict === "UNGRADED").length;
     const clean = outcomes.filter((o) => !o.suspect);
     const passes = clean.filter((o) => o.verdict === "PASS").length;
     const errored = outcomes.filter((o) => o.verdict === "ERROR").length;
@@ -50,9 +66,9 @@ export function aggregateReps(outcomes, threshold) {
         return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
     }
     const passRate = passes / clean.length;
-    const verdict = passRate >= threshold ? "PASS" : "FAIL";
+    const verdict = passRate >= threshold && (ungraded === 0 || passes > 0) ? "PASS" : ungraded > 0 ? "UNGRADED" : "FAIL";
     const flakiness = 1 - Math.abs(2 * passRate - 1);
-    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})`;
+    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})${ungraded ? `; ${ungraded} UNGRADED` : ""}`;
     return { verdict, reason, passes, reps, clean: clean.length, flakiness, suspect: false };
 }
 /**
@@ -63,6 +79,9 @@ export function aggregateReps(outcomes, threshold) {
  * caller to merge.
  */
 export function outcomesToResult(id, outcomes, repCount, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
+    const ungraded = outcomes.filter(outcome => outcome.verdict === "UNGRADED").length;
+    const ungradedField = ungraded ? { ungraded_reps: ungraded } : {};
     // Spread rather than always-set: a scenario with no trace assertions must
     // produce a result byte-identical to one from before this field existed.
     const objective = aggregateObjective(outcomes);
@@ -88,13 +107,13 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     const repJudgmentField = outcomes.some(outcome => outcome.judgment) ? { rep_judgments: repJudgments } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
+        pass_threshold: threshold, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
     };
 }
 function aggregateMetrics(outcomes) {

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, effectiveThreshold } from "./results.js";
+import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, normalizeScenarioResult, effectiveThreshold } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -47,7 +47,8 @@ export function rescoreRun(opts) {
         throw new Error(`no results.yaml in ${opts.runDir}`);
     const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
     const changes = [];
-    const scenarios = prev.scenarios.map((s) => {
+    const scenarios = prev.scenarios.map((recorded) => {
+        const s = normalizeScenarioResult(recorded);
         const scenario = specById.get(s.id);
         // no rate to re-apply, or an untrustworthy verdict → carry verbatim
         if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
@@ -61,7 +62,7 @@ export function rescoreRun(opts) {
         if (toThreshold === fromThreshold)
             return s;
         const rate = s.passes / s.clean;
-        const verdict = rate >= toThreshold ? "PASS" : "FAIL";
+        const verdict = rate >= toThreshold && (!s.ungraded_reps || s.passes > 0) ? "PASS" : s.ungraded_reps ? "UNGRADED" : "FAIL";
         if (verdict !== s.judge_verdict) {
             changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
         }
@@ -76,6 +77,14 @@ export function rescoreRun(opts) {
         // already measured. It re-measures nothing, so it may discard nothing.
         return rebuildScenarioResult({ ...s, judge_verdict: verdict, pass_threshold: toThreshold }, s, { objective: "carry", adjudication: "carry" });
     });
+    for (const s of scenarios) {
+        const recorded = prev.scenarios.find(prior => prior.id === s.id);
+        if (s.judge_verdict !== recorded.judge_verdict && !changes.some(change => change.id === s.id)) {
+            changes.push({ id: s.id, from: recorded.judge_verdict, to: s.judge_verdict,
+                passes: s.passes ?? (s.judge_verdict === "PASS" ? 1 : 0), clean: s.clean ?? 1,
+                fromThreshold: recorded.pass_threshold ?? 0.5, toThreshold: s.pass_threshold ?? 0.5 });
+        }
+    }
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         schema: prev.schema, subject_invocations: prev.subject_invocations,

package/dist/results.d.ts

@@ -61,6 +61,8 @@ export interface ScenarioResult {
     passes?: number;
     clean?: number;
     flakiness?: number;
+    /** Reps with incomplete criterion votes; included as non-passes, never misfires. */
+    ungraded_reps?: number;
     pass_threshold?: number;
     /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
     metrics?: ScenarioMetrics;
@@ -99,7 +101,9 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
-    /** One retry with identical judge input after unparseable criterion votes. */
+    /** Structured format for new judgments; prose is retained compatibility evidence. */
+    judgeFormat?: "json" | "prose";
+    /** One validation-guided retry after an invalid structured reply. */
     judgeRetries?: 1;
 }
 export interface AdjudicationResult {
@@ -349,6 +353,7 @@ export declare function resultsPath(runDir: string): string;
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export declare function normalizeScenarioResult(s: ScenarioResult): ScenarioResult;
 export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,

package/dist/results.js

@@ -5,6 +5,7 @@ import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
 import { collapseVotePanel } from "./vote-panel.js";
+import { outcomesToResult } from "./reps.js";
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export function mergeScenarioMetrics(prior, fresh) {
     if (!prior)
@@ -149,8 +150,28 @@ export function resultsPath(runDir) {
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export function normalizeScenarioResult(s) {
+    const panels = s.rep_judgments;
+    if (!panels?.some(panel => panel.recorded_verdict === "UNGRADED" || panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))))
+        return s;
+    const reps = s.reps ?? 1;
+    if (panels.length !== reps)
+        return { ...s, judge_verdict: "UNGRADED", suspect: false, ungraded_reps: panels.filter(panel => panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))).length };
+    const outcomes = [...panels].sort((a, b) => a.repetition - b.repetition).map(panel => ({
+        verdict: panel.recorded_verdict,
+        reason: panel.judgments[0]?.reason ?? s.judge_reason,
+        suspect: panel.recorded_verdict === "UNGRADED" ? false : panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")),
+        objective: panel.objective,
+        judgment: panel.judgments.find(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? panel.judgments[0],
+    }));
+    const fresh = outcomesToResult(s.id, outcomes, reps, s.pass_threshold ?? 0.5);
+    fresh.metrics = s.metrics;
+    fresh.usage = s.usage;
+    fresh.rep_judgments = panels.map(panel => ({ ...panel, recorded_verdict: fresh.rep_judgments?.find(candidate => candidate.repetition === panel.repetition)?.recorded_verdict ?? panel.recorded_verdict }));
+    return rebuildScenarioResult(fresh, s, { objective: "carry", adjudication: "carry" });
+}
 export function effectiveVerdicts(scenarios) {
-    return scenarios.map((s) => ({
+    return scenarios.map(normalizeScenarioResult).map((s) => ({
         id: s.id,
         verdict: s.override ?? objectiveVerdict(s) ?? s.judge_verdict,
         suspect: s.suspect && s.override == null, // an override resolves the misfire
@@ -182,9 +203,10 @@ function objectiveVerdict(s) {
  * deployment and is scored (see SCORED_MODES directly above).
  */
 export function finalizeResults(draft, ctx) {
+    const scenarios = draft.scenarios.map(normalizeScenarioResult);
     let effective_grade;
     if (ctx) {
-        const s = score(effectiveVerdicts(draft.scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
+        const s = score(effectiveVerdicts(scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
         effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
     }
     else {
@@ -214,7 +236,7 @@ export function finalizeResults(draft, ctx) {
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         ...(draft.source_hash_roots ? { source_hash_roots: draft.source_hash_roots } : {}),
         effective_grade,
-        scenarios: draft.scenarios,
+        scenarios,
         ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),
     };
 }
@@ -304,7 +326,11 @@ export function recomputeRecordedPanels(results) {
         for (const panel of scenario.rep_judgments ?? []) {
             const clean = panel.judgments.filter(j => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
             let verdict = panel.recorded_verdict;
-            if (clean.length === 1)
+            if (panel.objective && panel.objective.status !== "PASS")
+                verdict = panel.objective.status;
+            else if (panel.recorded_verdict === "UNGRADED" && panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")))
+                verdict = "UNGRADED";
+            else if (clean.length === 1)
                 verdict = clean[0].verdict;
             else if (clean.length >= 2)
                 verdict = collapseVotePanel(panel.judgments).verdict ?? "JUDGE-AMBIGUOUS";
@@ -417,7 +443,7 @@ export function validateResults(raw) {
             if (!Number.isInteger(scenario.adjudication.repetition) || scenario.adjudication.repetition < 0 || scenario.adjudication.repetition >= reps)
                 throw new Error(`schema v3 adjudication repetition missing or out of range for ${scenario.id}`);
             const collapsed = collapseVotePanel(scenario.adjudication.judgments);
-            const boundedCriticalAggregate = (scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
+            const boundedCriticalAggregate = ((scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 || scenario.judge_verdict === "UNGRADED") && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
             if (!boundedCriticalAggregate && (scenario.adjudication.state !== collapsed.state || scenario.adjudication.verdict !== collapsed.verdict))
                 throw new Error(`recorded adjudication state/verdict diverges from recomputed votes for ${scenario.id}`);
             if (scenario.adjudication.state !== "unresolved")
@@ -426,13 +452,14 @@ export function validateResults(raw) {
         const panels = scenario.rep_judgments;
         const errorCount = panels.filter(panel => panel.recorded_verdict === "ERROR").length;
         const notMeasuredCount = panels.filter(panel => panel.recorded_verdict === "NOT-MEASURED").length;
-        const cleanPanels = panels.filter(panel => !(panel.judgments[0]?.suspect ?? false));
+        const cleanPanels = panels.filter(panel => panel.objective && panel.objective.status !== "PASS" || panel.recorded_verdict === "UNGRADED" || !(panel.judgments[0]?.suspect ?? false));
+        const ungradedCount = panels.filter(panel => panel.recorded_verdict === "UNGRADED").length;
         const passes = cleanPanels.filter(panel => panel.recorded_verdict === "PASS").length;
         let aggregateVerdict;
         let aggregateSuspect = false;
         if (panels.length === 1) {
             aggregateVerdict = panels[0].recorded_verdict;
-            aggregateSuspect = panels[0].judgments[0]?.suspect ?? false;
+            aggregateSuspect = panels[0].objective && panels[0].objective.status !== "PASS" || panels[0].recorded_verdict === "UNGRADED" ? false : panels[0].judgments[0]?.suspect ?? false;
         }
         else if (errorCount > 0)
             aggregateVerdict = "ERROR";
@@ -443,7 +470,7 @@ export function validateResults(raw) {
             aggregateSuspect = true;
         }
         else
-            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) ? "PASS" : "FAIL";
+            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) && (ungradedCount === 0 || passes > 0) ? "PASS" : ungradedCount > 0 ? "UNGRADED" : "FAIL";
         const expectedVerdict = adjudicatedVerdict ?? aggregateVerdict;
         const expectedSuspect = scenario.adjudication ? scenario.adjudication.state === "unresolved" : aggregateSuspect;
         if (scenario.judge_verdict !== expectedVerdict || scenario.suspect !== expectedSuspect)
@@ -600,7 +627,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, ungraded_reps, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -613,7 +640,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
     const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
-        (pickedAdjudication.verdict === "PASS" && pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+        (pickedAdjudication.verdict === "PASS" && (judge_verdict === "UNGRADED" || pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS"))));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
     const adjudication = conflictsWithFreshEvidence && pickedAdjudication
@@ -652,6 +679,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(clean === undefined ? {} : { clean }),
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        ...(ungraded_reps === undefined ? {} : { ungraded_reps }),
         ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
         ...((freshUsage ?? prior?.usage) ? { usage: freshUsage ?? prior.usage } : {}),
         // The author owns the verdict; a re-measurement never discards their call.

package/dist/run.js

@@ -567,7 +567,8 @@ export function formatScorecard(summary, lift, stability) {
         const susp = s.suspect ? " ⚠suspect" : "";
         const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
         const repInfo = s.reps ? `  [${s.passes}/${s.clean}${misfired}${s.flakiness ? ` flaky ${s.flakiness.toFixed(2)}` : ""}]` : "";
-        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}`);
+        const ungradedInfo = s.ungraded_reps ? ` · ${s.ungraded_reps} ungraded rep${s.ungraded_reps === 1 ? "" : "s"} (UNGRADED)` : "";
+        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}${ungradedInfo}`);
     }
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";

package/dist/score.d.ts

@@ -3,7 +3,7 @@ import type { ShipBar } from "./spec.js";
  * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
  * never a pass and never silently resolved — it marks the run for a rejudge.
  */
-export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS";
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS" | "UNGRADED";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;
@@ -24,6 +24,7 @@ export interface ScoreResult {
     suspectCount: number;
     errorCount: number;
     notMeasuredCount: number;
+    ungradedCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;

package/dist/score.js

@@ -28,6 +28,7 @@ export function score(verdicts, input) {
     let suspectCount = 0;
     let errorCount = 0;
     let notMeasuredCount = 0;
+    let ungradedCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
@@ -46,6 +47,8 @@ export function score(verdicts, input) {
             passed++;
             continue;
         }
+        if (v.verdict === "UNGRADED")
+            ungradedCount++;
         if (critical.includes(v.id))
             criticalFails++;
         if (/^B/i.test(v.id))
@@ -75,12 +78,15 @@ export function score(verdicts, input) {
     else if (notMeasuredCount > 0) {
         note = `${notMeasuredCount} not measured: skill delivery was not established`;
     }
+    else if (ungradedCount > 0) {
+        note = `${ungradedCount} UNGRADED: incomplete criterion votes (counted as non-passes)`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, ungradedCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/stability.js

@@ -9,7 +9,7 @@ function conclusive(v) {
     // Same rule lift.ts applies, for the same reason: ERROR is a harness failure and an
     // unresolved misfire is a judge failure. Counting either as a side of a flip would
     // report infrastructure noise as behavioural instability.
-    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS";
+    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS" && v.verdict !== "UNGRADED";
 }
 function pointFor(r, s, verdict) {
     const reps = s.reps ?? 1;

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -112,12 +112,16 @@ export function collectTrends(skillDir, limit = 20) {
             // effectiveVerdicts is the single source of truth for the override-aware
             // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
             // resolves the misfire); zip in flakiness from the matching ScenarioResult.
-            const verdicts = effectiveVerdicts(r.scenarios);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
+            const verdicts = effectiveVerdicts(normalized);
             const cells = {};
-            r.scenarios.forEach((s, i) => {
+            normalized.forEach((s, i) => {
                 cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
             });
-            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade, cells });
         }
         models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.3",
+    "version": "0.23.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -102,7 +102,8 @@
         "PASS",
         "FAIL",
         "ERROR",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -232,6 +233,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
@@ -321,6 +323,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,

package/schemas/results-v3.schema.json

@@ -113,7 +113,8 @@
         "FAIL",
         "ERROR",
         "NOT-MEASURED",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -244,6 +245,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
@@ -347,6 +349,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,