@post-print/agent-harness 1.1.8 → 2.0.0

package/dist/judge.js

@@ -1,795 +0,0 @@
-import { missingClassifierAuth, runClassifier } from "./classifier.js";
-import { isTransientInfraError, resolveRetryMaxAttempts, withRetry } from "./retry.js";
-import { sumUsageParts } from "./usage-breakdown.js";
-// Optional info-string (`json`, `js`, `typescript`, …) on the opening fence line.
-// Without this, ` ```js\n"yes"\n``` ` extracts `js\n"yes"` and never latches.
-const JSON_FENCE_PATTERN = /```[^\n`]*\s*([\s\S]*?)```/;
-const JSON_OBJECT_PATTERN = /\{[\s\S]*\}/;
-const JUDGE_TOOL_RESULT_MAX_CHARS = 4_000;
-function truncateJudgeText(text, max = JUDGE_TOOL_RESULT_MAX_CHARS) {
-    if (text.length <= max) {
-        return text;
-    }
-    return `${text.slice(0, max)}…`;
-}
-function formatOneToolCallForJudge(call, index) {
-    const args = JSON.stringify(call.args ?? {});
-    const lines = [`${index + 1}. ${call.name} ${truncateJudgeText(args, 2_000)}`];
-    if (call.result) {
-        lines.push(`   result: ${truncateJudgeText(call.result)}`);
-    }
-    return lines.join("\n");
-}
-function formatToolCallsForJudge(trace) {
-    if (trace.toolCalls.length === 0) {
-        return "";
-    }
-    return [
-        "Tool calls:",
-        ...trace.toolCalls.map((call, index) => formatOneToolCallForJudge(call, index)),
-    ].join("\n");
-}
-function formatTraceChronologically(trace) {
-    const blocks = [];
-    let tie = 0;
-    for (const message of trace.messages) {
-        blocks.push({
-            seq: message.seq ?? Number.MAX_SAFE_INTEGER,
-            tie: tie++,
-            text: `${message.role}:\n${message.content}`,
-        });
-    }
-    for (const [index, call] of trace.toolCalls.entries()) {
-        blocks.push({
-            seq: call.seq ?? Number.MAX_SAFE_INTEGER,
-            tie: tie++,
-            text: `Tool calls:\n${formatOneToolCallForJudge(call, index)}`,
-        });
-    }
-    blocks.sort((a, b) => a.seq - b.seq || a.tie - b.tie);
-    return blocks.map((block) => block.text).join("\n\n---\n\n");
-}
-/** Full live transcript for a judge, including tool names, args, and results. */
-export function formatTraceForJudge(trace) {
-    const hasSeq = trace.messages.some((message) => message.seq !== undefined) ||
-        trace.toolCalls.some((call) => call.seq !== undefined);
-    const body = hasSeq
-        ? formatTraceChronologically(trace)
-        : [...trace.messages.map((m) => `${m.role}:\n${m.content}`), formatToolCallsForJudge(trace)]
-            .filter(Boolean)
-            .join("\n\n---\n\n");
-    return [
-        body,
-        trace.prBody ?? "",
-        trace.gitDiff ? `Git diff:\n${trace.gitDiff}` : "",
-        trace.shellCommands.length > 0 ? `Shell commands:\n${trace.shellCommands.join("\n")}` : "",
-    ]
-        .filter(Boolean)
-        .join("\n\n---\n\n");
-}
-/** Judge questions for mustInvokeSkill — replaces prose regex heuristics. */
-export function skillInvokeJudgeCriteria(skillNames) {
-    return skillNames.map((name) => ({
-        id: `invoke-skill:${name}`,
-        question: [
-            `Did the agent actually follow the "${name}" skill?`,
-            "Use tool calls, file reads, and later actions as evidence.",
-            "Naming the skill or using similar words is not enough.",
-            "A host skill load counts as obtaining the skill body.",
-            "Do not fail only because a confirmatory SKILL.md read happens after the reply.",
-        ].join(" "),
-    }));
-}
-function normalizeVerdict(value) {
-    const verdict = String(value ?? "")
-        .trim()
-        .toLowerCase();
-    if (verdict === "yes" || verdict === "no") {
-        return verdict;
-    }
-    return undefined;
-}
-// Whole-reply opening fence is a contract attempt regardless of info-string
-// (`json`, `js`, `typescript`, bare, …). Mid-prose fence mentions do not match.
-const JUDGE_JSON_FENCE_OPEN_PATTERN = /^```/;
-const JUDGE_VERDICT_KEY_PATTERN = /"verdict"\s*:/;
-/** Same prefixes legacy YES/NO strips — gate same-line JSON peels. */
-const JUDGE_VERDICT_PREFIX_PATTERN = /^(?:answer|verdict|line\s*1)\s*:\s*/i;
-/** Complete JSON number token (rejects bare `1.` with no fractional digits). */
-const JUDGE_JSON_NUMBER_TOKEN = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/;
-const JUDGE_JSON_BOOL_NULL_TOKEN = /^(?:true|false|null)/i;
-/** Closed fences, or a trailing unclosed fence (prose preamble + open contract). */
-const MARKDOWN_FENCE_STRIP_PATTERN = /```[^\n`]*\s*[\s\S]*?(?:```|$)/g;
-/**
- * After a complete JSON number/bool/null token: whether the remainder is
- * contract junk (latch) vs English continuation (allow YES/NO salvage).
- *
- * `true\nYES` / `3 YES` / `42,` / `3, YES` → latch.
- * `true story` / `3 findings` / `3, findings` / `true, the agent` → salvage.
- */
-function jsonPrimitiveRemainderIsContract(rest) {
-    if (rest === "") {
-        return true;
-    }
-    const restTrim = rest.trimStart();
-    if (restTrim === "") {
-        return true;
-    }
-    // Comma after a primitive is JSON-ish only when nothing English follows
-    // (`42,` / `3, YES`). List/prose continuations (`3, findings`, `1, 2, and 3`)
-    // must still reach YES/NO salvage.
-    if (restTrim.startsWith(",")) {
-        const afterComma = restTrim.slice(1).trimStart();
-        if (afterComma === "") {
-            return true;
-        }
-        if (/^(?:yes|no)\b/i.test(afterComma)) {
-            return true;
-        }
-        if (/[A-Za-z]/.test(afterComma)) {
-            return false;
-        }
-        return true;
-    }
-    const firstWord = /^([A-Za-z]+)/.exec(restTrim)?.[1]?.toLowerCase();
-    if (firstWord === "yes" || firstWord === "no") {
-        return true;
-    }
-    if (/^[A-Za-z]/.test(restTrim)) {
-        return false;
-    }
-    return true;
-}
-/**
- * Detect truncated/unparseable JSON-shaped contract attempts. Only replies
- * that *are* JSON-shaped (value prefix or open with a fence) count — a mid-prose
- * mention of ```json or an incidental blob is not a contract attempt.
- *
- * Number/bool/null prefixes latch only for a complete JSON token (optionally
- * with trailing junk). Numbered-list prose (`1. …`) and English continuations
- * (`true story`, `3 findings`) must not latch — YES/NO salvage still applies.
- */
-function looksLikeJudgeJsonAttempt(text) {
-    const trimmed = text.trim();
-    if (JUDGE_JSON_FENCE_OPEN_PATTERN.test(trimmed)) {
-        return true;
-    }
-    if (/^["{[]/.test(trimmed)) {
-        return true;
-    }
-    const number = JUDGE_JSON_NUMBER_TOKEN.exec(trimmed);
-    if (number) {
-        const rest = trimmed.slice(number[0].length);
-        // `1. prose` / `10) prose` — list markers, not number-then-junk.
-        if (/^[.)]/.test(rest)) {
-            return false;
-        }
-        return jsonPrimitiveRemainderIsContract(rest);
-    }
-    const boolNull = JUDGE_JSON_BOOL_NULL_TOKEN.exec(trimmed);
-    if (boolNull) {
-        return jsonPrimitiveRemainderIsContract(trimmed.slice(boolNull[0].length));
-    }
-    return false;
-}
-/** Drop fenced bodies so legacy YES/NO does not match inside quoted JSON strings. */
-function stripMarkdownFencesForLegacy(text) {
-    return text.replace(MARKDOWN_FENCE_STRIP_PATTERN, "\n");
-}
-/**
- * True when the JSON contract body is array-shaped (`[` before any `{`),
- * including truncated arrays. Object-extraction must not peel an inner
- * `{…}` out of those replies.
- */
-function isArrayShapedJudgeJson(text) {
-    const trimmed = text.trim();
-    const fenced = trimmed.match(JSON_FENCE_PATTERN);
-    const body = fenced?.[1]?.trim() ?? trimmed;
-    const bracket = body.indexOf("[");
-    if (bracket === -1) {
-        return false;
-    }
-    const brace = body.indexOf("{");
-    return brace === -1 || bracket < brace;
-}
-const INVALID_JUDGE_JSON = {
-    pass: false,
-    rationale: "judge returned invalid JSON (expected { verdict, evidence, rationale })",
-    evidence: [],
-    valid: false,
-};
-function arrayLooksLikeJudgeContract(parsed) {
-    if (parsed.some((item) => item !== null &&
-        typeof item === "object" &&
-        !Array.isArray(item) &&
-        "verdict" in item)) {
-        return true;
-    }
-    // Wrong-shaped contract answers like `["yes"]` / `["no"]` — not incidental
-    // lists such as `[1,2,3]` or `["intro","summary"]`.
-    return (parsed.length > 0 &&
-        parsed.every((item) => typeof item === "string" && normalizeVerdict(item) !== undefined));
-}
-/** Peeled objects with judge-schema keys but no `verdict` are contract attempts. */
-function objectLooksLikeJudgeSchema(record) {
-    return "evidence" in record || "rationale" in record;
-}
-/**
- * Try structured judge JSON. `structured` latches only for real contract
- * attempts: whole-text/fenced JSON bodies, verdict-shaped arrays or objects,
- * or failed candidates carrying a `verdict` key — not incidental arrays,
- * mid-prose truncated blobs, or fence mentions inside prose.
- */
-function tryParseJudgeJson(text) {
-    const trimmed = text.trim();
-    const candidates = [trimmed];
-    const fenced = trimmed.match(JSON_FENCE_PATTERN);
-    const fencedBody = fenced?.[1]?.trim();
-    if (fencedBody) {
-        candidates.unshift(fencedBody);
-    }
-    const arrayShaped = isArrayShapedJudgeJson(trimmed);
-    // Prose-prefixed single objects may still use greedy `{…}` extraction.
-    // Array-shaped replies (including truncated `[{…}`) must not peel inner objects.
-    if (!arrayShaped) {
-        const objectMatch = trimmed.match(JSON_OBJECT_PATTERN);
-        if (objectMatch?.[0]) {
-            candidates.push(objectMatch[0]);
-        }
-        const brace = trimmed.indexOf("{");
-        if (brace > 0) {
-            candidates.push(trimmed.slice(brace));
-        }
-    }
-    else {
-        const bracket = trimmed.indexOf("[");
-        if (bracket >= 0) {
-            candidates.push(trimmed.slice(bracket));
-        }
-    }
-    // Contract peels under a prose preamble (not mid-prose incidental quotes):
-    // - line-leading `"…` / `[…` (`Answer:\n"yes"`)
-    // - line-leading JSON number/bool/null (`Answer:\n42\nYES`)
-    // - same-line after answer|verdict|line 1: (`Answer: "yes"`, `Verdict: 42`)
-    // Mid-line quotes without that prefix (`YES` + ```js\n"yes"\n```) stay
-    // incidental — fence bodies use `fencedBody` / whole-reply fence primary.
-    const lineLeadingJsonValues = new Set();
-    let insideFence = false;
-    const pushContractPeel = (value) => {
-        const peel = value.trim();
-        if (!peel) {
-            return;
-        }
-        candidates.push(peel);
-        lineLeadingJsonValues.add(peel);
-    };
-    const isLineLeadingJsonPrimitive = (lineTrim) => {
-        if (lineTrim.startsWith('"') || lineTrim.startsWith("[")) {
-            return true;
-        }
-        // Complete number/bool/null lines (optional contract junk) — not English
-        // (`3 findings`, `true story`) which looksLikeJudgeJsonAttempt rejects.
-        return (looksLikeJudgeJsonAttempt(lineTrim) &&
-            (JUDGE_JSON_NUMBER_TOKEN.test(lineTrim) || JUDGE_JSON_BOOL_NULL_TOKEN.test(lineTrim)));
-    };
-    for (const line of trimmed.split("\n")) {
-        const lineTrim = line.trim();
-        if (lineTrim.startsWith("```")) {
-            insideFence = !insideFence;
-            continue;
-        }
-        if (insideFence) {
-            continue;
-        }
-        if (isLineLeadingJsonPrimitive(lineTrim)) {
-            pushContractPeel(lineTrim);
-        }
-        const prefix = JUDGE_VERDICT_PREFIX_PATTERN.exec(lineTrim);
-        if (prefix) {
-            const remainder = lineTrim.slice(prefix[0].length).trim();
-            if (remainder.startsWith('"') ||
-                remainder.startsWith("[") ||
-                remainder.startsWith("{") ||
-                isLineLeadingJsonPrimitive(remainder)) {
-                pushContractPeel(remainder);
-            }
-        }
-    }
-    let structured = false;
-    let failedJsonShape = false;
-    // Only whole-reply fences are primary contract attempts. Mid-prose fences
-    // (e.g. YES + ```js\n"yes"\n```) stay incidental like unfenced peels.
-    const wholeReplyFence = JUDGE_JSON_FENCE_OPEN_PATTERN.test(trimmed);
-    for (const candidate of candidates) {
-        const candidateTrim = candidate.trim();
-        const primary = candidateTrim === trimmed ||
-            (wholeReplyFence && fencedBody !== undefined && candidateTrim === fencedBody);
-        const lineLeading = lineLeadingJsonValues.has(candidateTrim);
-        try {
-            const parsed = JSON.parse(candidate);
-            if (parsed === null || typeof parsed !== "object") {
-                // Whole-text/fenced JSON primitives (`"yes"`, `42`, `true`, `null`)
-                // are contract attempts that violate the object contract — refuse
-                // and latch so callers do not YES/NO-salvage a quoted verdict.
-                // Preamble peels (`Answer:\n"yes"`, `Answer: "yes"`, `Answer:\n42`)
-                // are the same wrong-shaped contract; mid-line incidental quote
-                // peels stay out of `lineLeadingJsonValues`.
-                if (primary || lineLeading) {
-                    return { result: { ...INVALID_JUDGE_JSON }, structured: true };
-                }
-                continue;
-            }
-            if (Array.isArray(parsed)) {
-                // Whole-text/fenced arrays are contract attempts; peeled arrays count
-                // when verdict-shaped or yes/no string arrays (`["yes"]`). Incidental
-                // lists (e.g. `Scores: [1,2,3]`) still allow YES/NO salvage.
-                if (primary || arrayLooksLikeJudgeContract(parsed)) {
-                    return { result: { ...INVALID_JUDGE_JSON }, structured: true };
-                }
-                continue;
-            }
-            const record = parsed;
-            // Objects without `verdict`: schema-shaped peels (`evidence`/`rationale`)
-            // are failed contract attempts; bare quote blobs stay incidental.
-            if (!("verdict" in record)) {
-                if (primary || objectLooksLikeJudgeSchema(record)) {
-                    return { result: { ...INVALID_JUDGE_JSON }, structured: true };
-                }
-                continue;
-            }
-            structured = true;
-            const verdict = normalizeVerdict(record.verdict);
-            if (verdict !== undefined) {
-                return {
-                    result: {
-                        pass: verdict === "yes",
-                        rationale: String(record.rationale ?? "").trim() || "no rationale",
-                        evidence: Array.isArray(record.evidence)
-                            ? record.evidence.map((item) => String(item)).filter(Boolean)
-                            : [],
-                        valid: true,
-                    },
-                    structured: true,
-                };
-            }
-            return { result: { ...INVALID_JUDGE_JSON }, structured: true };
-        }
-        catch {
-            // A failed parse latches `structured` only for real contract attempts:
-            // the whole/fenced body is a JSON value (object/array/primitive),
-            // including trailing junk after a primitive (`"yes" clearly`), or a
-            // peeled candidate carries a `"verdict":` key (truncated contract).
-            // Line-leading string/array peels (`Answer:\n"yes" clearly`,
-            // `Answer:\n["yes"`) latch the same way. Mid-prose incidental peels
-            // (e.g. `Evidence: {"quote":"hello"`) keep YES/NO salvage available.
-            if (primary) {
-                if (looksLikeJudgeJsonAttempt(candidateTrim)) {
-                    failedJsonShape = true;
-                }
-            }
-            else if ((candidateTrim.startsWith("{") || candidateTrim.startsWith("[")) &&
-                JUDGE_VERDICT_KEY_PATTERN.test(candidateTrim)) {
-                failedJsonShape = true;
-            }
-            else if (lineLeading && looksLikeJudgeJsonAttempt(candidateTrim)) {
-                failedJsonShape = true;
-            }
-        }
-    }
-    return {
-        result: { ...INVALID_JUDGE_JSON },
-        structured: structured || failedJsonShape || looksLikeJudgeJsonAttempt(trimmed),
-    };
-}
-/** Parse structured judge JSON (`verdict`, `evidence`, `rationale`). */
-export function parseJudgeJsonResponse(text) {
-    return tryParseJudgeJson(text).result;
-}
-const JUDGE_MARKDOWN_TRIM_PATTERN = /^\*+\s*|\s*\*+$/g;
-const JUDGE_TRAILING_PUNCT_PATTERN = /[.:!]+$/g;
-const JUDGE_TAIL_VERDICT_PATTERN = /\b(YES|NO)\.?\s*$/i;
-const JUDGE_INLINE_VERDICT_PATTERN = /\b(YES|NO)\b/i;
-function extractLegacyVerdict(line) {
-    const stripped = line
-        .replace(JUDGE_MARKDOWN_TRIM_PATTERN, "")
-        .replace(JUDGE_VERDICT_PREFIX_PATTERN, "")
-        .trim();
-    const head = stripped.replace(JUDGE_TRAILING_PUNCT_PATTERN, "").toUpperCase();
-    if (head === "YES" || head.startsWith("YES ")) {
-        return "yes";
-    }
-    if (head === "NO" || head.startsWith("NO ")) {
-        return "no";
-    }
-    const tailMatch = stripped.match(JUDGE_TAIL_VERDICT_PATTERN);
-    if (tailMatch?.[1]) {
-        return tailMatch[1].toLowerCase();
-    }
-    const inlineMatch = stripped.match(JUDGE_INLINE_VERDICT_PATTERN);
-    if (inlineMatch?.[1]) {
-        return inlineMatch[1].toLowerCase();
-    }
-    return undefined;
-}
-/** Fallback when model ignores JSON contract — parse YES/NO prose. */
-export function parseJudgeLegacyResponse(text) {
-    const lines = text
-        .trim()
-        .split("\n")
-        .map((line) => line.trim())
-        .filter(Boolean);
-    for (let index = 0; index < Math.min(lines.length, 3); index++) {
-        const line = lines[index];
-        if (!line) {
-            continue;
-        }
-        const verdict = extractLegacyVerdict(line);
-        if (verdict !== undefined) {
-            const rationale = lines
-                .filter((_, i) => i !== index)
-                .join(" ")
-                .trim() || line.replace(JUDGE_TAIL_VERDICT_PATTERN, "").trim();
-            return {
-                pass: verdict === "yes",
-                rationale: rationale || line,
-                evidence: [],
-                valid: true,
-            };
-        }
-    }
-    const fallback = extractLegacyVerdict(lines[0] ?? text.trim());
-    if (fallback !== undefined) {
-        return {
-            pass: fallback === "yes",
-            rationale: lines.slice(1).join(" ").trim() || lines[0] || text,
-            evidence: [],
-            valid: true,
-        };
-    }
-    return {
-        pass: false,
-        rationale: "judge returned invalid JSON (expected { verdict, evidence, rationale })",
-        evidence: [],
-        valid: false,
-    };
-}
-/**
- * Parse structured JSON first, then legacy YES/NO prose.
- * Once a real JSON contract attempt is detected (array, object with `verdict`,
- * or truncated JSON-shaped text), never salvage YES/NO from the body — invalid
- * contracts are infra failures, not rubric answers.
- */
-export function parseJudgeResponse(text) {
-    const { result, structured } = tryParseJudgeJson(text);
-    if (result.valid || structured) {
-        return result;
-    }
-    // Strip fences so preamble + ```json\n"yes"\n``` cannot salvage from the
-    // quoted primitive; outer YES/NO beside incidental fences still works.
-    return parseJudgeLegacyResponse(stripMarkdownFencesForLegacy(text));
-}
-function buildJudgePrompt(transcript, question, workspaces) {
-    return [
-        "You are a test harness classifier. You may inspect the explicitly listed read-only workspace evidence with file-read tools or read-only shell commands such as cat, sed, rg, ls, and stat. Shell commands may only inspect those evidence paths. Do not execute project code, edit files, install packages, use network access, or inspect any other path.",
-        "Decide whether the transcript and workspace evidence satisfy the criterion.",
-        "The transcript includes assistant text, tool names, tool args, and tool results.",
-        "A tool result is an outcome. Use it as evidence when the criterion asks about one.",
-        "Reply with one JSON object only — no markdown fences, no text before or after:",
-        '{"verdict":"yes"|"no","evidence":["exact direct quote from transcript"],"rationale":"one sentence"}',
-        "Evidence must contain one or more exact, contiguous excerpts copied from the transcript in double quotes.",
-        "Do not paraphrase, summarize, infer, or invent evidence. If no exact quote supports the verdict, return an empty evidence array.",
-        'Use verdict "yes" only when evidence clearly supports the criterion.',
-        ...formatJudgeWorkspaces(workspaces),
-        "",
-        "Transcript:",
-        transcript,
-        "",
-        `Criterion: ${question}`,
-    ].join("\n");
-}
-function formatJudgeWorkspaces(workspaces) {
-    if (!workspaces?.length)
-        return ["Workspace evidence: none was provided."];
-    return [
-        "Read-only workspace evidence (use only these paths; each path is immutable):",
-        ...workspaces.map((workspace) => `- ${workspace.name}: ${workspace.path}`),
-    ];
-}
-export function buildCompareJudgePrompt(options) {
-    return [
-        "You are a test harness classifier. You may inspect the explicitly listed read-only workspace evidence with file-read tools or read-only shell commands such as cat, sed, rg, ls, and stat. Shell commands may only inspect those evidence paths. Do not execute project code, edit files, install packages, use network access, or inspect any other path.",
-        "Decide whether the transcripts and workspace evidence satisfy the criterion.",
-        "Arm A and arm B are two agent runs of the same test.",
-        "The transcripts include assistant text, tool names, tool args, and tool results.",
-        "A tool result is an outcome. Use it as evidence when the criterion asks about one.",
-        "Reply with one JSON object only — no markdown fences, no text before or after:",
-        '{"verdict":"yes"|"no","evidence":["exact direct quote from a transcript"],"rationale":"one sentence"}',
-        "Evidence must contain one or more exact, contiguous excerpts copied from a transcript in double quotes.",
-        "Do not paraphrase, summarize, infer, or invent evidence. If no exact quote supports the verdict, return an empty evidence array.",
-        'Use verdict "yes" only when evidence clearly supports the criterion.',
-        ...formatJudgeWorkspaces(options.workspaces),
-        "",
-        `Arm A (${options.aLabel}):`,
-        options.aTranscript,
-        "",
-        `Arm B (${options.bLabel}):`,
-        options.bTranscript,
-        "",
-        `Criterion: ${options.question}`,
-    ].join("\n");
-}
-export function buildMultiArmCompareJudgePrompt(options) {
-    const armBlocks = options.arms.flatMap((arm) => [`Arm ${arm.label}:`, arm.transcript, ""]);
-    return [
-        "You are a test harness classifier. You may inspect the explicitly listed read-only workspace evidence with file-read tools or read-only shell commands such as cat, sed, rg, ls, and stat. Shell commands may only inspect those evidence paths. Do not execute project code, edit files, install packages, use network access, or inspect any other path.",
-        "Decide whether the transcripts and workspace evidence satisfy the criterion.",
-        "Each arm is one agent run of the same test.",
-        "Do not pick a single winner unless the criterion asks for one.",
-        "The transcripts include assistant text, tool names, tool args, and tool results.",
-        "A tool result is an outcome. Use it as evidence when the criterion asks about one.",
-        "Reply with one JSON object only — no markdown fences, no text before or after:",
-        '{"verdict":"yes"|"no","evidence":["exact direct quote from a transcript"],"rationale":"one sentence"}',
-        "Evidence must contain one or more exact, contiguous excerpts copied from a transcript in double quotes.",
-        "Do not paraphrase, summarize, infer, or invent evidence. If no exact quote supports the verdict, return an empty evidence array.",
-        'Use verdict "yes" only when evidence clearly supports the criterion.',
-        ...formatJudgeWorkspaces(options.workspaces),
-        "",
-        ...armBlocks,
-        `Criterion: ${options.question}`,
-    ].join("\n");
-}
-function formatJudgeInfraError(result) {
-    const details = [];
-    if (result.rawStatus) {
-        details.push(`sdk: ${result.rawStatus}`);
-    }
-    if (result.sdkError?.code) {
-        details.push(`code: ${result.sdkError.code}`);
-    }
-    if (result.sdkError?.message) {
-        details.push(`error: ${result.sdkError.message}`);
-    }
-    if (details.length === 0) {
-        return `judge run status: ${result.status}`;
-    }
-    return `judge run status: ${result.status} (${details.join(", ")})`;
-}
-async function runJudgePromptOnce(prompt, options) {
-    const started = performance.now();
-    const host = options.host ?? "cursor";
-    const classify = options.classify ?? runClassifier;
-    const result = await classify({
-        host,
-        cwd: options.workspaces?.[0]?.path ?? options.cwd,
-        prompt,
-        apiKey: options.apiKey,
-        workspaces: options.workspaces,
-        onText: options.onText,
-    });
-    const durationMs = Math.round(performance.now() - started);
-    const usage = result.usage;
-    if (result.status !== "completed") {
-        const infraError = formatJudgeInfraError(result);
-        return {
-            pass: false,
-            rationale: infraError,
-            evidence: [],
-            error: infraError,
-            infraError,
-            rawSdkStatus: result.rawStatus ?? result.status,
-            sdkError: result.sdkError,
-            durationMs,
-            usage,
-            retryable: isTransientInfraError(infraError),
-        };
-    }
-    const parsed = parseJudgeResponse(result.text);
-    if (!parsed.valid) {
-        const rationale = `${parsed.rationale} (raw: ${result.text.slice(0, 160) || "empty"})`;
-        return {
-            pass: false,
-            rationale,
-            evidence: [],
-            response: result.text,
-            error: rationale,
-            parseError: rationale,
-            durationMs,
-            usage,
-            retryable: false,
-        };
-    }
-    return {
-        pass: parsed.pass,
-        rationale: parsed.rationale,
-        evidence: parsed.evidence,
-        response: result.text,
-        durationMs,
-        usage,
-        retryable: false,
-    };
-}
-async function runJudgePrompt(prompt, options) {
-    const host = options.host ?? "cursor";
-    const missing = missingClassifierAuth(host, options.apiKey);
-    if (missing) {
-        return {
-            pass: false,
-            rationale: missing,
-            evidence: [],
-            error: "missing api key",
-            infraError: "missing api key",
-            durationMs: 0,
-            attempt: 1,
-        };
-    }
-    const maxAttempts = resolveRetryMaxAttempts();
-    const started = performance.now();
-    let lastAttempt = 1;
-    try {
-        const { result, attempt } = await withRetry(async (attemptNumber) => {
-            lastAttempt = attemptNumber;
-            const outcome = await runJudgePromptOnce(prompt, options);
-            if (outcome.retryable && attemptNumber < maxAttempts) {
-                throw new Error(outcome.infraError ?? outcome.error ?? "judge infra error");
-            }
-            return outcome;
-        }, {
-            maxAttempts,
-            shouldRetry: (error) => isTransientInfraError(error instanceof Error ? error.message : String(error)),
-        });
-        return {
-            ...result,
-            durationMs: Math.round(performance.now() - started),
-            attempt,
-        };
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : "judge prompt failed";
-        return {
-            pass: false,
-            rationale: message,
-            evidence: [],
-            error: message,
-            infraError: message,
-            durationMs: Math.round(performance.now() - started),
-            attempt: lastAttempt,
-        };
-    }
-}
-/** Score fuzzy rubric criteria with a structured JSON judge (live runs only). */
-export async function judgeTrace(trace, criteria, options) {
-    if (criteria.length === 0) {
-        return { verdicts: [], skipped: true };
-    }
-    const host = options.host ?? "cursor";
-    const missing = missingClassifierAuth(host, options.apiKey);
-    if (missing) {
-        return {
-            verdicts: [],
-            skipped: true,
-            error: `${missing} — judge criteria skipped`,
-        };
-    }
-    const transcript = formatTraceForJudge(trace);
-    return judgeCriteria(criteria, (question) => buildJudgePrompt(transcript, question, options.workspaces), options, {
-        transcriptChars: transcript.length,
-        workspaceEvidence: options.workspaces?.map((workspace) => workspace.name),
-    });
-}
-function compareJudgeArms(input) {
-    if ("arms" in input) {
-        return input.arms;
-    }
-    return [
-        { label: input.aLabel, trace: input.a },
-        { label: input.bLabel, trace: input.b },
-    ];
-}
-function isCompareJudgePair(input) {
-    return "a" in input && "b" in input;
-}
-/** Score criteria against every arm transcript. */
-export async function judgeCompareTraces(input, criteria, options) {
-    if (criteria.length === 0) {
-        return { verdicts: [], skipped: true };
-    }
-    const host = options.host ?? "cursor";
-    const missing = missingClassifierAuth(host, options.apiKey);
-    if (missing) {
-        return {
-            verdicts: [],
-            skipped: true,
-            error: `${missing} — judge criteria skipped`,
-        };
-    }
-    const pair = isCompareJudgePair(input)
-        ? input
-        : input.arms.length === 2 && input.arms[0] && input.arms[1]
-            ? {
-                aLabel: input.arms[0].label,
-                a: input.arms[0].trace,
-                bLabel: input.arms[1].label,
-                b: input.arms[1].trace,
-            }
-            : undefined;
-    if (pair) {
-        const aTranscript = formatTraceForJudge(pair.a);
-        const bTranscript = formatTraceForJudge(pair.b);
-        return judgeCriteria(criteria, (question) => buildCompareJudgePrompt({
-            aLabel: pair.aLabel,
-            aTranscript,
-            bLabel: pair.bLabel,
-            bTranscript,
-            question,
-            workspaces: options.workspaces,
-        }), options, {
-            transcriptChars: aTranscript.length + bTranscript.length,
-            workspaceEvidence: options.workspaces?.map((workspace) => workspace.name),
-        });
-    }
-    const arms = compareJudgeArms(input).map((arm) => ({
-        label: arm.label,
-        transcript: formatTraceForJudge(arm.trace),
-    }));
-    return judgeCriteria(criteria, (question) => buildMultiArmCompareJudgePrompt({ arms, question, workspaces: options.workspaces }), options, {
-        transcriptChars: arms.reduce((sum, arm) => sum + arm.transcript.length, 0),
-        workspaceEvidence: options.workspaces?.map((workspace) => workspace.name),
-    });
-}
-async function judgeCriteria(criteria, buildPrompt, options, meta) {
-    const verdicts = [];
-    for (const criterion of criteria) {
-        const prompt = buildPrompt(criterion.question);
-        options.onEvent?.({
-            type: "criterion_started",
-            id: criterion.id,
-            question: criterion.question,
-        });
-        const parsed = await runJudgePrompt(prompt, {
-            ...options,
-            onText: (text) => {
-                options.onText?.(text);
-                options.onEvent?.({
-                    type: "text",
-                    id: criterion.id,
-                    question: criterion.question,
-                    text,
-                });
-            },
-        });
-        const verdict = {
-            id: criterion.id,
-            pass: parsed.pass,
-            rationale: parsed.rationale,
-            evidence: parsed.evidence,
-            prompt,
-            response: parsed.response,
-            infraError: parsed.infraError,
-            parseError: parsed.parseError,
-            rawSdkStatus: parsed.rawSdkStatus,
-            sdkError: parsed.sdkError,
-            attempt: parsed.attempt,
-            durationMs: parsed.durationMs,
-            usage: parsed.usage,
-            transcriptChars: meta.transcriptChars,
-            promptChars: prompt.length,
-            workspaceEvidence: meta.workspaceEvidence,
-        };
-        verdicts.push(verdict);
-        options.onEvent?.({ type: "criterion_finished", question: criterion.question, verdict });
-        if (parsed.error) {
-            return {
-                verdicts,
-                skipped: false,
-                error: parsed.error,
-                usage: sumUsageParts(verdicts.map((v) => v.usage)),
-            };
-        }
-    }
-    return {
-        verdicts,
-        skipped: false,
-        usage: sumUsageParts(verdicts.map((v) => v.usage)),
-    };
-}
-//# sourceMappingURL=judge.js.map