@skill-harness/core 0.22.4 → 0.23.0

package/dist/grade.d.ts

@@ -36,13 +36,23 @@ export declare function parseVerdict(out: string): ParsedVerdict;
  */
 export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
 export interface GradeResult extends ParsedVerdict {
+    /** The final judge reply, retained for callers that predate structured judging. */
     raw: string;
+    /** Every attempt in call order; two entries mean the structured retry also ran. */
+    rawReplies: string[];
     criteria: CriterionVote[];
-    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeFormat: "json";
+    /** One validation-guided retry after an invalid structured reply. Absent on the first attempt. */
     judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export declare function parseStructuredJudgeReply(raw: string, expectedCriteria: number): ParsedVerdict & {
+    criteria: CriterionVote[];
+};
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export declare function formatJudgeRawReplies(replies: string[]): string;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
  * the overall verdict equals AND(items). A mismatch in EITHER direction — verdict
@@ -51,11 +61,11 @@ export interface GradeResult extends ParsedVerdict {
  * parse, or the verdict is ERROR, return false (never block a run on a parse miss).
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
-/** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
+/** Drive the judge for one transcript and parse the JSON-only result. */
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
+import { completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -37,9 +37,8 @@ ${numbered}
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
 ${diffGuidance}
-Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
-VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
-REASON: <15 words or fewer>`;
+Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Reply with exactly one JSON object and nothing else, with one vote per criterion in checklist order:
+{"votes":[{"criterion":1,"vote":"PASS"|"FAIL","reason":"..."},...],"verdict":"PASS"|"FAIL","reason":"..."}`;
 }
 // Both anchor to the start of a line and REQUIRE the colon. Without those anchors the
 // reason pattern matched any word containing "reason" — git-ops GLM C1 stored
@@ -87,6 +86,85 @@ export function judgeResemblesSubject(judge, subject) {
     const b = subject.model;
     return a === b || a.includes(b) || b.includes(a);
 }
+/** Extract the first complete top-level JSON object, tolerating prose or fences around it. */
+function extractFirstJsonObject(raw) {
+    const start = raw.indexOf("{");
+    if (start < 0)
+        throw new Error("no JSON object in judge reply");
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === "\\")
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"')
+            inString = true;
+        else if (ch === "{")
+            depth++;
+        else if (ch === "}" && --depth === 0)
+            return raw.slice(start, i + 1);
+    }
+    throw new Error("incomplete JSON object in judge reply");
+}
+function exactKeys(value, expected, context) {
+    const actual = Object.keys(value).sort();
+    const wanted = [...expected].sort();
+    if (JSON.stringify(actual) !== JSON.stringify(wanted)) {
+        throw new Error(`${context} must contain exactly ${wanted.join(", ")}`);
+    }
+}
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export function parseStructuredJudgeReply(raw, expectedCriteria) {
+    let value;
+    try {
+        value = JSON.parse(extractFirstJsonObject(raw));
+    }
+    catch (error) {
+        if (error instanceof Error && /JSON object in judge reply/.test(error.message))
+            throw error;
+        throw new Error(`invalid JSON: ${error.message}`);
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error("judge reply must be a JSON object");
+    const object = value;
+    exactKeys(object, ["votes", "verdict", "reason"], "judge reply");
+    if (!Array.isArray(object.votes))
+        throw new Error("votes must be an array");
+    if (object.votes.length !== expectedCriteria)
+        throw new Error(`expected ${expectedCriteria} votes, got ${object.votes.length}`);
+    const criteria = object.votes.map((rawVote, offset) => {
+        if (!rawVote || typeof rawVote !== "object" || Array.isArray(rawVote))
+            throw new Error(`vote ${offset + 1} must be an object`);
+        const vote = rawVote;
+        exactKeys(vote, ["criterion", "vote", "reason"], `vote ${offset + 1}`);
+        if (vote.criterion !== offset + 1)
+            throw new Error(`vote ${offset + 1} must have criterion ${offset + 1}`);
+        if (vote.vote !== "PASS" && vote.vote !== "FAIL")
+            throw new Error(`vote ${offset + 1} must be PASS or FAIL`);
+        if (typeof vote.reason !== "string")
+            throw new Error(`vote ${offset + 1} reason must be a string`);
+        return { index: offset + 1, verdict: vote.vote, reason: vote.reason };
+    });
+    if (object.verdict !== "PASS" && object.verdict !== "FAIL")
+        throw new Error("verdict must be PASS or FAIL");
+    if (typeof object.reason !== "string")
+        throw new Error("reason must be a string");
+    return { criteria, verdict: object.verdict, reason: object.reason };
+}
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export function formatJudgeRawReplies(replies) {
+    if (replies.length === 1)
+        return replies[0];
+    return replies.map((reply, index) => `=== JUDGE REPLY ${index + 1} ===\n${reply}`).join("\n\n");
+}
 const ITEM_RE = /^\s*\d+[.)]\s*\**\s*(PASS|FAIL)\b/gim;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
@@ -122,27 +200,48 @@ export function detectMisfire(raw, verdict) {
 }
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
-/** Drive the judge for one transcript and parse the result. */
+/** Drive the judge for one transcript and parse the JSON-only result. */
 export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
-    const request = { model: judge, prompt, cwd };
-    let raw = await adapter.judge(request);
-    let criteria = parseCriterionVotes(raw);
-    let judgeRetries;
-    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
-        raw = await adapter.judge(request);
-        criteria = parseCriterionVotes(raw);
-        judgeRetries = 1;
-    }
-    const parsed = parseVerdict(raw);
-    // On a parse failure, surface what the judge actually emitted (e.g. a provider
-    // error) rather than a generic message — otherwise the cause is invisible.
-    if (parsed.verdict === "ERROR") {
-        const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
-        if (snippet)
-            parsed.reason = `judge unparseable: ${snippet}`;
+    const rawReplies = [];
+    let validationError = "";
+    for (let attempt = 0; attempt < 2; attempt++) {
+        const retry = attempt === 0 ? "" : `\n\nYour reply was not valid: ${JSON.stringify(validationError)}; reply with only the JSON object.`;
+        const raw = await adapter.judge({ model: judge, prompt: prompt + retry, cwd });
+        rawReplies.push(raw);
+        // Pi adapter failures are infrastructure errors, not malformed model votes.
+        if (/^\[judge error:/i.test(raw.trim())) {
+            const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
+            return {
+                verdict: "ERROR", reason: `judge unparseable: ${snippet}`, suspect: false,
+                raw, rawReplies, criteria: completeCriterionVotes([], expectedCriteria), judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        try {
+            const parsed = parseStructuredJudgeReply(raw, expectedCriteria);
+            const suspect = parsed.verdict === "FAIL"
+                ? parsed.criteria.every(vote => vote.verdict === "PASS")
+                : parsed.criteria.some(vote => vote.verdict === "FAIL");
+            return {
+                ...parsed, raw, rawReplies, suspect, judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        catch (error) {
+            validationError = error.message;
+        }
     }
-    const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
+    const raw = rawReplies[rawReplies.length - 1];
+    return {
+        verdict: "UNGRADED",
+        reason: `judge structured reply invalid after retry: ${validationError}`,
+        suspect: false,
+        raw,
+        rawReplies,
+        criteria: completeCriterionVotes([], expectedCriteria),
+        judgeFormat: "json",
+        judgeRetries: 1,
+    };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the

package/dist/regate.js

@@ -49,11 +49,19 @@ function rewriteTranscript(path, gateLines) {
     renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
     writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
 }
-/** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
-function verdictFromSavedJudgement(runDir, id, mode, rep) {
+/** Recover a rep's judge verdict from retained structured evidence or its legacy prose artifact. */
+function verdictFromSavedJudgement(runDir, id, mode, rep, retained) {
     const path = judgeRawPath(runDir, id, mode, rep);
     if (!existsSync(path))
         return null;
+    if (retained?.judgeFormat === "json") {
+        return {
+            verdict: retained.verdict,
+            reason: retained.reason,
+            suspect: retained.suspect,
+            judgment: retained,
+        };
+    }
     const raw = readFileSync(path, "utf8");
     const parsed = parseVerdict(raw);
     return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
@@ -284,7 +292,9 @@ export async function regateRun(opts) {
             if (!oldGateFailed) {
                 // The judge already saw this rep. Its verdict is on disk — re-read it rather
                 // than paying to ask the same question again.
-                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
+                const panel = rec.rep_judgments?.find(candidate => candidate.repetition === (rep ?? 0));
+                const retained = panel?.judgments.find(judgment => judgment.ordinal === 1) ?? panel?.judgments[0];
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep, retained);
                 outcomes.push({ ...(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect }), objective });
                 continue;
             }

package/dist/regrade.js

@@ -1,6 +1,6 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
+import { buildJudgePrompt, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
 import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
 import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
@@ -57,10 +57,10 @@ export async function judgeOneRep(opts) {
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
-    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
+    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), formatJudgeRawReplies(g.rawReplies), "utf8");
     const outcome = normalizeRepOutcome({
         verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),

package/dist/results.d.ts

@@ -101,7 +101,9 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
-    /** One retry with identical judge input after unparseable criterion votes. */
+    /** Structured format for new judgments; prose is retained compatibility evidence. */
+    judgeFormat?: "json" | "prose";
+    /** One validation-guided retry after an invalid structured reply. */
     judgeRetries?: 1;
 }
 export interface AdjudicationResult {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.4",
+    "version": "0.23.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -233,6 +233,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",

package/schemas/results-v3.schema.json

@@ -245,6 +245,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",