@skill-harness/core 0.22.4 → 0.24.0

package/dist/adapters/types.d.ts

@@ -78,6 +78,8 @@ export interface StructuredRun {
      * answering badly. `run.ts` turns this into ERROR — never a model verdict.
      */
     providerFailure?: string;
+    /** Terminal error/abort or missing settlement: unavailable outcome, never a judgeable success. */
+    executionFailure?: string;
 }
 export interface HarnessAdapter {
     name: string;
@@ -104,7 +106,11 @@ export interface HarnessAdapter {
      * field is to identify which CLI produced a transcript.
      *
      * Optional so a test double or a future adapter need not implement it; callers
-     * treat a missing method exactly like a null answer.
+     * treat a missing method exactly like a null answer. A qualification probe must
+     * use the same cwd and environment as the execution it qualifies.
      */
-    version?(): Promise<string | null>;
+    version?(options?: {
+        cwd?: string;
+        env?: NodeJS.ProcessEnv;
+    }): Promise<string | null>;
 }

package/dist/capture-trace-types.d.ts

@@ -92,6 +92,7 @@ export interface TraceMetrics {
     /** Date of the repository price table used for cost_usd; null when unpriced. */
     price_as_of: string | null;
     tool_calls: number;
+    /** Legacy field: requested Agent arguments, not verified child launches. */
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */
     max_concurrency: number;
@@ -118,6 +119,8 @@ export interface ExecutionTraceV1 {
      * never contained, changing grades on scenarios nobody edited.
      */
     final_text: string;
+    /** Observed terminal state; unknown versions are unqualified, historical Pi 0.83.0 has no settlement field. */
+    final_status?: "complete" | "error" | "aborted" | "incomplete" | "truncated" | "unavailable" | "unqualified";
     /** Every tool call in this invocation, in issue order. */
     tool_calls: TraceToolCall[];
     /**

package/dist/execution-trace.d.ts

@@ -1,5 +1,26 @@
 import type { ExecutionTraceV1 } from "./capture-trace-types.js";
 import type { ModelRef, RunMode } from "./adapters/types.js";
+/**
+ * Parse pi's `--mode json` event stream into an `ExecutionTraceV1`.
+ *
+ * Pinned by the fixtures in `packages/adapters/test/fixtures/pi-json/`. Three
+ * event-shape constraints are baked in here and each one is a bug if removed:
+ *
+ * 1. **Line-at-a-time, never buffered.** `message_update` re-sends the entire
+ *    accumulated message on every delta, so the stream is quadratic in output
+ *    length — a trivial three-tool-call run emitted 52 MB wrapping 12 KB of
+ *    terminal events. The parser consumes an iterable of lines and holds only
+ *    what it keeps.
+ * 2. **The final assistant message only.** pi's print mode — what the judge has
+ *    always been shown — emits exactly that. Concatenating every assistant text
+ *    block would add interim narration the transcript has never contained and
+ *    move verdicts on scenarios nobody edited.
+ * 3. **Correlate tool calls by `toolCallId`.** Batched calls execute
+ *    concurrently and their `end` events arrive in completion order.
+ */
+/** Current execution is qualified only against this exact Pi release. */
+export declare const QUALIFIED_PI_VERSION = "1.0.4";
+export declare function piQualificationFailure(version: string | null): string | null;
 export interface TraceMeta {
     piVersion: string | null;
     subject: ModelRef;

package/dist/execution-trace.js

@@ -20,6 +20,14 @@ import { redactArgs, redactText } from "./redaction.js";
  * 3. **Correlate tool calls by `toolCallId`.** Batched calls execute
  *    concurrently and their `end` events arrive in completion order.
  */
+/** Current execution is qualified only against this exact Pi release. */
+export const QUALIFIED_PI_VERSION = "1.0.4";
+/** Historical raw captures from this version remain readable; this is not fresh-run qualification. */
+const LEGACY_PI_TRACE_VERSION = "0.83.0";
+export function piQualificationFailure(version) {
+    return version === QUALIFIED_PI_VERSION ? null
+        : `Pi ${version ?? "(version unavailable)"} is unqualified; execution requires exact Pi ${QUALIFIED_PI_VERSION}`;
+}
 /** Events that carry no information a trace keeps, and are large. */
 const SKIPPED = new Set(["message_update", "tool_execution_update"]);
 /** `details` larger than this is dropped rather than persisted. */
@@ -40,6 +48,13 @@ export function parseTrace(lines, meta) {
     let completionCounter = 0;
     let malformedLines = 0;
     let sawTerminal = false;
+    const settledContract = meta.piVersion !== LEGACY_PI_TRACE_VERSION;
+    const qualificationFailure = settledContract ? piQualificationFailure(meta.piVersion) : null;
+    let sawSettled = false;
+    let lastStopReason;
+    let currentAssistantText = "";
+    let eligibleFinal = false;
+    const captureErrors = [];
     let finalText = "";
     let lastAssistantText = "";
     let inputTokens = null;
@@ -60,13 +75,35 @@ export function parseTrace(lines, meta) {
             malformedLines++;
             continue;
         }
+        if (!ev || typeof ev !== "object" || Array.isArray(ev)) {
+            malformedLines++;
+            continue;
+        }
         const type = ev.type;
-        if (typeof type !== "string" || SKIPPED.has(type))
+        if (typeof type !== "string") {
+            malformedLines++;
+            continue;
+        }
+        if (settledContract && ["agent_start", "turn_start", "message_start", "message_update", "message_end", "tool_execution_start", "tool_execution_end"].includes(type)) {
+            sawSettled = false;
+            if (type !== "tool_execution_end") {
+                currentAssistantText = "";
+                lastStopReason = undefined;
+                eligibleFinal = false;
+            }
+        }
+        if (SKIPPED.has(type))
             continue;
         if (type === "tool_execution_start") {
             const id = str(ev.toolCallId);
-            if (!id)
+            if (!id) {
+                captureErrors.push("tool start without an identity");
                 continue;
+            }
+            if (calls.has(id)) {
+                captureErrors.push("duplicate tool start identity");
+                continue;
+            }
             calls.set(id, {
                 id,
                 name: str(ev.toolName) ?? "(unknown)",
@@ -83,11 +120,19 @@ export function parseTrace(lines, meta) {
         }
         if (type === "tool_execution_end") {
             const id = str(ev.toolCallId);
-            if (!id)
+            if (!id) {
+                captureErrors.push("tool end without an identity");
                 continue;
+            }
             const call = calls.get(id);
-            if (!call)
-                continue; // an end with no start is not evidence of a call
+            if (!call) {
+                captureErrors.push("tool end without a matching start");
+                continue;
+            }
+            if (call.completionIndex >= 0) {
+                captureErrors.push("duplicate tool completion identity");
+                continue;
+            }
             if (call.completionIndex < 0)
                 activeCalls = Math.max(0, activeCalls - 1);
             call.completionIndex = completionCounter++;
@@ -100,7 +145,20 @@ export function parseTrace(lines, meta) {
         if (type === "message_end") {
             sawTerminal = true;
             const msg = ev.message;
-            const at = isoTime(msg?.timestamp);
+            if (!msg || typeof msg !== "object" || Array.isArray(msg) || typeof msg.role !== "string") {
+                captureErrors.push("malformed message_end message");
+                continue;
+            }
+            // Pi also emits system/user messages; a system message has string content.
+            // They invalidate a previous final but are not assistant/tool-result payloads.
+            if (msg.role !== "assistant" && msg.role !== "toolResult")
+                continue;
+            if (!Array.isArray(msg.content) ||
+                msg.content.some((block) => !block || typeof block !== "object" || Array.isArray(block) || typeof block.type !== "string" || (block.type === "text" && typeof block.text !== "string"))) {
+                captureErrors.push("malformed message_end content");
+                continue;
+            }
+            const at = isoTime(msg.timestamp);
             if (msg?.role === "assistant" && at) {
                 for (const block of msg.content ?? []) {
                     if (block.type !== "toolCall" || typeof block.id !== "string")
@@ -120,6 +178,9 @@ export function parseTrace(lines, meta) {
             if (msg?.role !== "assistant")
                 continue;
             const text = assistantText(msg);
+            eligibleFinal = msg.content.every((block) => block.type === "text" || block.type === "thinking");
+            currentAssistantText = text;
+            lastStopReason = msg.stopReason;
             if (text) {
                 lastAssistantText = text;
                 // `stop` marks the model's closing message; a `toolUse` message is
@@ -135,6 +196,8 @@ export function parseTrace(lines, meta) {
         }
         if (type === "turn_end" || type === "agent_end" || type === "agent_settled") {
             sawTerminal = true;
+            if (type === "agent_settled")
+                sawSettled = true;
             // Deliberately read NOTHING from these. They repeat the same assistant
             // messages `message_end` already carried (`turn_end` and `agent_end` do;
             // `agent_settled` carries no keys at all beyond `type`), and reading two
@@ -143,6 +206,23 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
+    if (malformedLines > 0)
+        captureErrors.push(`pi JSONL contained ${malformedLines} malformed line(s); trace evidence is incomplete`);
+    const finalStatus = qualificationFailure ? "unqualified"
+        : !sawSettled || activeCalls > 0 ? "incomplete"
+            : captureErrors.length > 0 ? "unavailable"
+                : lastStopReason === "error" ? "error"
+                    : lastStopReason === "aborted" ? "aborted"
+                        : lastStopReason === "length" ? "truncated"
+                            : lastStopReason === "stop" && eligibleFinal && currentAssistantText.length > 0 ? "complete" : "unavailable";
+    if (settledContract && finalStatus !== "complete") {
+        const reason = qualificationFailure
+            ?? (finalStatus === "incomplete" ? "current settlement is missing or tool calls remain outstanding"
+                : finalStatus === "truncated" ? "settled assistant reached its output length limit"
+                    : finalStatus === "error" || finalStatus === "aborted" ? `settled assistant stop reason is ${lastStopReason}`
+                        : "current terminal assistant content is empty, ineligible, or captured evidence is malformed");
+        captureErrors.push(`Pi ${meta.piVersion ?? "(version unavailable)"} final delivery is ${finalStatus}; ${reason}`);
+    }
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
@@ -172,7 +252,9 @@ export function parseTrace(lines, meta) {
         // and `smoke-real-pi.sh` asserts no `/home/` survives into a persisted trace
         // — an assertion that used to pass only because the smoke model happened not
         // to echo one.
-        final_text: redactText(finalText || lastAssistantText, meta.homeDir),
+        final_text: redactText(settledContract ? currentAssistantText : finalText || lastAssistantText, meta.homeDir),
+        ...(settledContract ? { final_status: finalStatus } : {}),
+        ...(captureErrors.length ? { capture_errors: [...new Set(captureErrors)] } : {}),
         tool_calls: toolCalls,
         // `null`, not `[]`: the stream says nothing about the filesystem. The runner
         // overwrites this after observing the workspace. Defaulting to `[]` claimed
@@ -184,7 +266,7 @@ export function parseTrace(lines, meta) {
         metrics,
     };
     trace.trace_sha256 = traceSha256(trace);
-    return { trace, isComplete: sawTerminal, malformedLines };
+    return { trace, isComplete: settledContract ? !qualificationFailure && sawSettled && activeCalls === 0 : sawTerminal, malformedLines };
 }
 function addReported(current, value) {
     return typeof value === "number" && value > 0 ? (current ?? 0) + value : current;
@@ -194,8 +276,7 @@ function assistantText(msg) {
     return (msg.content ?? [])
         .filter((b) => b.type === "text" && typeof b.text === "string")
         .map((b) => b.text)
-        .join("\n")
-        .trim();
+        .join("");
 }
 /**
  * Bounded metadata about a tool result. The body is never kept: results carry

package/dist/grade.d.ts

@@ -36,13 +36,23 @@ export declare function parseVerdict(out: string): ParsedVerdict;
  */
 export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
 export interface GradeResult extends ParsedVerdict {
+    /** The final judge reply, retained for callers that predate structured judging. */
     raw: string;
+    /** Every attempt in call order; two entries mean the structured retry also ran. */
+    rawReplies: string[];
     criteria: CriterionVote[];
-    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeFormat: "json";
+    /** One validation-guided retry after an invalid structured reply. Absent on the first attempt. */
     judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export declare function parseStructuredJudgeReply(raw: string, expectedCriteria: number): ParsedVerdict & {
+    criteria: CriterionVote[];
+};
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export declare function formatJudgeRawReplies(replies: string[]): string;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
  * the overall verdict equals AND(items). A mismatch in EITHER direction — verdict
@@ -51,11 +61,11 @@ export interface GradeResult extends ParsedVerdict {
  * parse, or the verdict is ERROR, return false (never block a run on a parse miss).
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
-/** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
+/** Drive the judge for one transcript and parse the JSON-only result. */
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
+import { completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -37,9 +37,8 @@ ${numbered}
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
 ${diffGuidance}
-Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
-VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
-REASON: <15 words or fewer>`;
+Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Reply with exactly one JSON object and nothing else, with one vote per criterion in checklist order:
+{"votes":[{"criterion":1,"vote":"PASS"|"FAIL","reason":"..."},...],"verdict":"PASS"|"FAIL","reason":"..."}`;
 }
 // Both anchor to the start of a line and REQUIRE the colon. Without those anchors the
 // reason pattern matched any word containing "reason" — git-ops GLM C1 stored
@@ -87,6 +86,85 @@ export function judgeResemblesSubject(judge, subject) {
     const b = subject.model;
     return a === b || a.includes(b) || b.includes(a);
 }
+/** Extract the first complete top-level JSON object, tolerating prose or fences around it. */
+function extractFirstJsonObject(raw) {
+    const start = raw.indexOf("{");
+    if (start < 0)
+        throw new Error("no JSON object in judge reply");
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === "\\")
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"')
+            inString = true;
+        else if (ch === "{")
+            depth++;
+        else if (ch === "}" && --depth === 0)
+            return raw.slice(start, i + 1);
+    }
+    throw new Error("incomplete JSON object in judge reply");
+}
+function exactKeys(value, expected, context) {
+    const actual = Object.keys(value).sort();
+    const wanted = [...expected].sort();
+    if (JSON.stringify(actual) !== JSON.stringify(wanted)) {
+        throw new Error(`${context} must contain exactly ${wanted.join(", ")}`);
+    }
+}
+/** Parse and validate the JSON-only contract used for new judge calls. */
+export function parseStructuredJudgeReply(raw, expectedCriteria) {
+    let value;
+    try {
+        value = JSON.parse(extractFirstJsonObject(raw));
+    }
+    catch (error) {
+        if (error instanceof Error && /JSON object in judge reply/.test(error.message))
+            throw error;
+        throw new Error(`invalid JSON: ${error.message}`);
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error("judge reply must be a JSON object");
+    const object = value;
+    exactKeys(object, ["votes", "verdict", "reason"], "judge reply");
+    if (!Array.isArray(object.votes))
+        throw new Error("votes must be an array");
+    if (object.votes.length !== expectedCriteria)
+        throw new Error(`expected ${expectedCriteria} votes, got ${object.votes.length}`);
+    const criteria = object.votes.map((rawVote, offset) => {
+        if (!rawVote || typeof rawVote !== "object" || Array.isArray(rawVote))
+            throw new Error(`vote ${offset + 1} must be an object`);
+        const vote = rawVote;
+        exactKeys(vote, ["criterion", "vote", "reason"], `vote ${offset + 1}`);
+        if (vote.criterion !== offset + 1)
+            throw new Error(`vote ${offset + 1} must have criterion ${offset + 1}`);
+        if (vote.vote !== "PASS" && vote.vote !== "FAIL")
+            throw new Error(`vote ${offset + 1} must be PASS or FAIL`);
+        if (typeof vote.reason !== "string")
+            throw new Error(`vote ${offset + 1} reason must be a string`);
+        return { index: offset + 1, verdict: vote.vote, reason: vote.reason };
+    });
+    if (object.verdict !== "PASS" && object.verdict !== "FAIL")
+        throw new Error("verdict must be PASS or FAIL");
+    if (typeof object.reason !== "string")
+        throw new Error("reason must be a string");
+    return { criteria, verdict: object.verdict, reason: object.reason };
+}
+/** Preserve one raw reply byte-for-byte, or both retry attempts with explicit boundaries. */
+export function formatJudgeRawReplies(replies) {
+    if (replies.length === 1)
+        return replies[0];
+    return replies.map((reply, index) => `=== JUDGE REPLY ${index + 1} ===\n${reply}`).join("\n\n");
+}
 const ITEM_RE = /^\s*\d+[.)]\s*\**\s*(PASS|FAIL)\b/gim;
 /**
  * Judge-misfire detector: parse the judge's per-checklist-item grades and assert
@@ -122,27 +200,48 @@ export function detectMisfire(raw, verdict) {
 }
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
-/** Drive the judge for one transcript and parse the result. */
+/** Drive the judge for one transcript and parse the JSON-only result. */
 export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
-    const request = { model: judge, prompt, cwd };
-    let raw = await adapter.judge(request);
-    let criteria = parseCriterionVotes(raw);
-    let judgeRetries;
-    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
-        raw = await adapter.judge(request);
-        criteria = parseCriterionVotes(raw);
-        judgeRetries = 1;
-    }
-    const parsed = parseVerdict(raw);
-    // On a parse failure, surface what the judge actually emitted (e.g. a provider
-    // error) rather than a generic message — otherwise the cause is invisible.
-    if (parsed.verdict === "ERROR") {
-        const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
-        if (snippet)
-            parsed.reason = `judge unparseable: ${snippet}`;
+    const rawReplies = [];
+    let validationError = "";
+    for (let attempt = 0; attempt < 2; attempt++) {
+        const retry = attempt === 0 ? "" : `\n\nYour reply was not valid: ${JSON.stringify(validationError)}; reply with only the JSON object.`;
+        const raw = await adapter.judge({ model: judge, prompt: prompt + retry, cwd });
+        rawReplies.push(raw);
+        // Pi adapter failures are infrastructure errors, not malformed model votes.
+        if (/^\[judge error:/i.test(raw.trim())) {
+            const snippet = raw.trim().replace(/\s+/g, " ").slice(0, 160);
+            return {
+                verdict: "ERROR", reason: `judge unparseable: ${snippet}`, suspect: false,
+                raw, rawReplies, criteria: completeCriterionVotes([], expectedCriteria), judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        try {
+            const parsed = parseStructuredJudgeReply(raw, expectedCriteria);
+            const suspect = parsed.verdict === "FAIL"
+                ? parsed.criteria.every(vote => vote.verdict === "PASS")
+                : parsed.criteria.some(vote => vote.verdict === "FAIL");
+            return {
+                ...parsed, raw, rawReplies, suspect, judgeFormat: "json",
+                ...(attempt === 1 ? { judgeRetries: 1 } : {}),
+            };
+        }
+        catch (error) {
+            validationError = error.message;
+        }
     }
-    const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
+    const raw = rawReplies[rawReplies.length - 1];
+    return {
+        verdict: "UNGRADED",
+        reason: `judge structured reply invalid after retry: ${validationError}`,
+        suspect: false,
+        raw,
+        rawReplies,
+        criteria: completeCriterionVotes([], expectedCriteria),
+        judgeFormat: "json",
+        judgeRetries: 1,
+    };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the

package/dist/provider-failure.d.ts

@@ -31,6 +31,10 @@
  * convention cannot drift apart.
  */
 export declare const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/** Failed or unproven terminal delivery, without assuming a provider-side cause. */
+export declare const EXECUTION_FAILURE_MARKER = "[skill-harness] execution failure:";
+export declare function withExecutionFailure(transcript: string, failure: string | null): string;
+export declare function executionFailureFromTranscript(transcript: string): string | null;
 /**
  * Prefix `transcript` with the provider-failure marker, or return it unchanged.
  *

package/dist/provider-failure.js

@@ -31,6 +31,14 @@
  * convention cannot drift apart.
  */
 export const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/** Failed or unproven terminal delivery, without assuming a provider-side cause. */
+export const EXECUTION_FAILURE_MARKER = "[skill-harness] execution failure:";
+export function withExecutionFailure(transcript, failure) {
+    return failure ? `${EXECUTION_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
+}
+export function executionFailureFromTranscript(transcript) {
+    return failureFromPreamble(transcript, EXECUTION_FAILURE_MARKER);
+}
 /**
  * The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),
  * which is where the preamble ends and model-influenced content begins.
@@ -87,11 +95,14 @@ export function providerFailureFromJsonLine(line) {
  * provider failed — it is evidence the model typed the words.
  */
 export function providerFailureFromTranscript(transcript) {
+    return failureFromPreamble(transcript, PROVIDER_FAILURE_MARKER);
+}
+function failureFromPreamble(transcript, marker) {
     for (const line of transcript.split("\n")) {
         if (line.startsWith(TURN_HEADER_PREFIX))
             return null;
-        if (line.startsWith(PROVIDER_FAILURE_MARKER))
-            return line.slice(PROVIDER_FAILURE_MARKER.length).trim();
+        if (line.startsWith(marker))
+            return line.slice(marker.length).trim();
     }
     return null;
 }

package/dist/regate.js

@@ -49,11 +49,19 @@ function rewriteTranscript(path, gateLines) {
     renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
     writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
 }
-/** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
-function verdictFromSavedJudgement(runDir, id, mode, rep) {
+/** Recover a rep's judge verdict from retained structured evidence or its legacy prose artifact. */
+function verdictFromSavedJudgement(runDir, id, mode, rep, retained) {
     const path = judgeRawPath(runDir, id, mode, rep);
     if (!existsSync(path))
         return null;
+    if (retained?.judgeFormat === "json") {
+        return {
+            verdict: retained.verdict,
+            reason: retained.reason,
+            suspect: retained.suspect,
+            judgment: retained,
+        };
+    }
     const raw = readFileSync(path, "utf8");
     const parsed = parseVerdict(raw);
     return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
@@ -284,7 +292,9 @@ export async function regateRun(opts) {
             if (!oldGateFailed) {
                 // The judge already saw this rep. Its verdict is on disk — re-read it rather
                 // than paying to ask the same question again.
-                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
+                const panel = rec.rep_judgments?.find(candidate => candidate.repetition === (rep ?? 0));
+                const retained = panel?.judgments.find(judgment => judgment.ordinal === 1) ?? panel?.judgments[0];
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep, retained);
                 outcomes.push({ ...(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect }), objective });
                 continue;
             }

package/dist/regrade.js

@@ -1,11 +1,11 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
+import { buildJudgePrompt, formatJudgeRawReplies, judgeInWorkspace } from "./grade.js";
 import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
 import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
-import { providerFailureFromTranscript } from "./provider-failure.js";
+import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -47,8 +47,9 @@ export async function judgeOneRep(opts) {
     // into a model verdict, spending a judge call to turn infrastructure noise into
     // a false FAIL.
     const providerFailure = providerFailureFromTranscript(transcript);
-    if (providerFailure) {
-        const reason = `provider failure — ${providerFailure}`;
+    const executionFailure = executionFailureFromTranscript(transcript);
+    if (providerFailure || executionFailure) {
+        const reason = providerFailure ? `provider failure — ${providerFailure}` : `execution failure — ${executionFailure}`;
         appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
         return {
             verdict: "ERROR", reason, suspect: false,
@@ -57,10 +58,10 @@ export async function judgeOneRep(opts) {
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
-    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
+    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), formatJudgeRawReplies(g.rawReplies), "utf8");
     const outcome = normalizeRepOutcome({
         verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), judgeFormat: g.judgeFormat, ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),

package/dist/results.d.ts

@@ -101,7 +101,9 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
-    /** One retry with identical judge input after unparseable criterion votes. */
+    /** Structured format for new judgments; prose is retained compatibility evidence. */
+    judgeFormat?: "json" | "prose";
+    /** One validation-guided retry after an invalid structured reply. */
     judgeRetries?: 1;
 }
 export interface AdjudicationResult {

package/dist/run.js

@@ -16,7 +16,7 @@ import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
-import { providerFailureFromTranscript } from "./provider-failure.js";
+import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
 /**
  * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
  * `PI_GRANTS_LEDGER` at exactly this filename inside the run directory. A fixed
@@ -303,6 +303,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                 // and the artifact disagreeing, and a recovered measurement thrown away.
                 infrastructureFailure = null;
                 adapterFailure = null;
+                let executionUnavailable = false;
                 try {
                     if (scenario.mode === "seeded") {
                         const r = await runSeeded(scenario, {
@@ -348,8 +349,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                             const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
                             transcript = structured.transcript;
                             traces = structured.traces;
+                            executionUnavailable = Boolean(structured.executionFailure);
                             if (structured.providerFailure)
                                 infrastructureFailure = `provider failure — ${structured.providerFailure}`;
+                            else if (structured.executionFailure)
+                                infrastructureFailure = `execution failure — ${structured.executionFailure}`;
                         }
                         else {
                             transcript = await ctx.adapter.run(req);
@@ -384,9 +388,19 @@ async function runRep(scenario, rep, repCount, ctx) {
                     const provider = providerFailureFromTranscript(transcript);
                     if (provider)
                         infrastructureFailure = `provider failure — ${provider}`;
+                    else {
+                        const execution = executionFailureFromTranscript(transcript);
+                        if (execution)
+                            infrastructureFailure = `execution failure — ${execution}`;
+                    }
                 }
-                noResponse = hasEmptyAssistantTurn(transcript);
-                if (!noResponse && !adapterFailure)
+                executionUnavailable ||= executionFailureFromTranscript(transcript) !== null;
+                // Qualified complete traces prove delivered bytes even if those bytes
+                // are only whitespace; the legacy text-only timeout heuristic cannot erase them.
+                const deliveredText = traces.length > 0 && traces.every((trace) => trace.final_status === "complete" && trace.final_text.length > 0 && !trace.capture_errors?.length);
+                noResponse = !deliveredText && hasEmptyAssistantTurn(transcript);
+                // Known unavailable execution is not an empty model answer to retry.
+                if (executionUnavailable || (!noResponse && !adapterFailure))
                     break;
             }
             // Survived the retry: ERROR for this rep, inside a run that still completes
@@ -578,7 +592,7 @@ export function formatScorecard(summary, lift, stability) {
         const tokens = metrics.input_tokens === null
             ? "subject tokens unavailable"
             : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
-        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
+        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} requested Agent task(s), max outstanding tool calls ${metrics.max_concurrency ?? 0}; child launches/concurrency unobserved`;
         const cost = metrics.cost_source === "subscription"
             ? " · subscription ($0 marginal cost recorded)"
             : metrics.cost_source === "unreported"

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.4",
+    "version": "0.24.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -233,6 +233,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",

package/schemas/results-v3.schema.json

@@ -245,6 +245,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",