@skill-harness/core 0.23.0 → 0.24.0

package/dist/adapters/types.d.ts

@@ -78,6 +78,8 @@ export interface StructuredRun {
      * answering badly. `run.ts` turns this into ERROR — never a model verdict.
      */
     providerFailure?: string;
+    /** Terminal error/abort or missing settlement: unavailable outcome, never a judgeable success. */
+    executionFailure?: string;
 }
 export interface HarnessAdapter {
     name: string;
@@ -104,7 +106,11 @@ export interface HarnessAdapter {
      * field is to identify which CLI produced a transcript.
      *
      * Optional so a test double or a future adapter need not implement it; callers
-     * treat a missing method exactly like a null answer.
+     * treat a missing method exactly like a null answer. A qualification probe must
+     * use the same cwd and environment as the execution it qualifies.
      */
-    version?(): Promise<string | null>;
+    version?(options?: {
+        cwd?: string;
+        env?: NodeJS.ProcessEnv;
+    }): Promise<string | null>;
 }

package/dist/capture-trace-types.d.ts

@@ -92,6 +92,7 @@ export interface TraceMetrics {
     /** Date of the repository price table used for cost_usd; null when unpriced. */
     price_as_of: string | null;
     tool_calls: number;
+    /** Legacy field: requested Agent arguments, not verified child launches. */
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */
     max_concurrency: number;
@@ -118,6 +119,8 @@ export interface ExecutionTraceV1 {
      * never contained, changing grades on scenarios nobody edited.
      */
     final_text: string;
+    /** Observed terminal state; unknown versions are unqualified, historical Pi 0.83.0 has no settlement field. */
+    final_status?: "complete" | "error" | "aborted" | "incomplete" | "truncated" | "unavailable" | "unqualified";
     /** Every tool call in this invocation, in issue order. */
     tool_calls: TraceToolCall[];
     /**

package/dist/execution-trace.d.ts

@@ -1,5 +1,26 @@
 import type { ExecutionTraceV1 } from "./capture-trace-types.js";
 import type { ModelRef, RunMode } from "./adapters/types.js";
+/**
+ * Parse pi's `--mode json` event stream into an `ExecutionTraceV1`.
+ *
+ * Pinned by the fixtures in `packages/adapters/test/fixtures/pi-json/`. Three
+ * event-shape constraints are baked in here and each one is a bug if removed:
+ *
+ * 1. **Line-at-a-time, never buffered.** `message_update` re-sends the entire
+ *    accumulated message on every delta, so the stream is quadratic in output
+ *    length — a trivial three-tool-call run emitted 52 MB wrapping 12 KB of
+ *    terminal events. The parser consumes an iterable of lines and holds only
+ *    what it keeps.
+ * 2. **The final assistant message only.** pi's print mode — what the judge has
+ *    always been shown — emits exactly that. Concatenating every assistant text
+ *    block would add interim narration the transcript has never contained and
+ *    move verdicts on scenarios nobody edited.
+ * 3. **Correlate tool calls by `toolCallId`.** Batched calls execute
+ *    concurrently and their `end` events arrive in completion order.
+ */
+/** Current execution is qualified only against this exact Pi release. */
+export declare const QUALIFIED_PI_VERSION = "1.0.4";
+export declare function piQualificationFailure(version: string | null): string | null;
 export interface TraceMeta {
     piVersion: string | null;
     subject: ModelRef;

package/dist/execution-trace.js

@@ -20,6 +20,14 @@ import { redactArgs, redactText } from "./redaction.js";
  * 3. **Correlate tool calls by `toolCallId`.** Batched calls execute
  *    concurrently and their `end` events arrive in completion order.
  */
+/** Current execution is qualified only against this exact Pi release. */
+export const QUALIFIED_PI_VERSION = "1.0.4";
+/** Historical raw captures from this version remain readable; this is not fresh-run qualification. */
+const LEGACY_PI_TRACE_VERSION = "0.83.0";
+export function piQualificationFailure(version) {
+    return version === QUALIFIED_PI_VERSION ? null
+        : `Pi ${version ?? "(version unavailable)"} is unqualified; execution requires exact Pi ${QUALIFIED_PI_VERSION}`;
+}
 /** Events that carry no information a trace keeps, and are large. */
 const SKIPPED = new Set(["message_update", "tool_execution_update"]);
 /** `details` larger than this is dropped rather than persisted. */
@@ -40,6 +48,13 @@ export function parseTrace(lines, meta) {
     let completionCounter = 0;
     let malformedLines = 0;
     let sawTerminal = false;
+    const settledContract = meta.piVersion !== LEGACY_PI_TRACE_VERSION;
+    const qualificationFailure = settledContract ? piQualificationFailure(meta.piVersion) : null;
+    let sawSettled = false;
+    let lastStopReason;
+    let currentAssistantText = "";
+    let eligibleFinal = false;
+    const captureErrors = [];
     let finalText = "";
     let lastAssistantText = "";
     let inputTokens = null;
@@ -60,13 +75,35 @@ export function parseTrace(lines, meta) {
             malformedLines++;
             continue;
         }
+        if (!ev || typeof ev !== "object" || Array.isArray(ev)) {
+            malformedLines++;
+            continue;
+        }
         const type = ev.type;
-        if (typeof type !== "string" || SKIPPED.has(type))
+        if (typeof type !== "string") {
+            malformedLines++;
+            continue;
+        }
+        if (settledContract && ["agent_start", "turn_start", "message_start", "message_update", "message_end", "tool_execution_start", "tool_execution_end"].includes(type)) {
+            sawSettled = false;
+            if (type !== "tool_execution_end") {
+                currentAssistantText = "";
+                lastStopReason = undefined;
+                eligibleFinal = false;
+            }
+        }
+        if (SKIPPED.has(type))
             continue;
         if (type === "tool_execution_start") {
             const id = str(ev.toolCallId);
-            if (!id)
+            if (!id) {
+                captureErrors.push("tool start without an identity");
                 continue;
+            }
+            if (calls.has(id)) {
+                captureErrors.push("duplicate tool start identity");
+                continue;
+            }
             calls.set(id, {
                 id,
                 name: str(ev.toolName) ?? "(unknown)",
@@ -83,11 +120,19 @@ export function parseTrace(lines, meta) {
         }
         if (type === "tool_execution_end") {
             const id = str(ev.toolCallId);
-            if (!id)
+            if (!id) {
+                captureErrors.push("tool end without an identity");
                 continue;
+            }
             const call = calls.get(id);
-            if (!call)
-                continue; // an end with no start is not evidence of a call
+            if (!call) {
+                captureErrors.push("tool end without a matching start");
+                continue;
+            }
+            if (call.completionIndex >= 0) {
+                captureErrors.push("duplicate tool completion identity");
+                continue;
+            }
             if (call.completionIndex < 0)
                 activeCalls = Math.max(0, activeCalls - 1);
             call.completionIndex = completionCounter++;
@@ -100,7 +145,20 @@ export function parseTrace(lines, meta) {
         if (type === "message_end") {
             sawTerminal = true;
             const msg = ev.message;
-            const at = isoTime(msg?.timestamp);
+            if (!msg || typeof msg !== "object" || Array.isArray(msg) || typeof msg.role !== "string") {
+                captureErrors.push("malformed message_end message");
+                continue;
+            }
+            // Pi also emits system/user messages; a system message has string content.
+            // They invalidate a previous final but are not assistant/tool-result payloads.
+            if (msg.role !== "assistant" && msg.role !== "toolResult")
+                continue;
+            if (!Array.isArray(msg.content) ||
+                msg.content.some((block) => !block || typeof block !== "object" || Array.isArray(block) || typeof block.type !== "string" || (block.type === "text" && typeof block.text !== "string"))) {
+                captureErrors.push("malformed message_end content");
+                continue;
+            }
+            const at = isoTime(msg.timestamp);
             if (msg?.role === "assistant" && at) {
                 for (const block of msg.content ?? []) {
                     if (block.type !== "toolCall" || typeof block.id !== "string")
@@ -120,6 +178,9 @@ export function parseTrace(lines, meta) {
             if (msg?.role !== "assistant")
                 continue;
             const text = assistantText(msg);
+            eligibleFinal = msg.content.every((block) => block.type === "text" || block.type === "thinking");
+            currentAssistantText = text;
+            lastStopReason = msg.stopReason;
             if (text) {
                 lastAssistantText = text;
                 // `stop` marks the model's closing message; a `toolUse` message is
@@ -135,6 +196,8 @@ export function parseTrace(lines, meta) {
         }
         if (type === "turn_end" || type === "agent_end" || type === "agent_settled") {
             sawTerminal = true;
+            if (type === "agent_settled")
+                sawSettled = true;
             // Deliberately read NOTHING from these. They repeat the same assistant
             // messages `message_end` already carried (`turn_end` and `agent_end` do;
             // `agent_settled` carries no keys at all beyond `type`), and reading two
@@ -143,6 +206,23 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
+    if (malformedLines > 0)
+        captureErrors.push(`pi JSONL contained ${malformedLines} malformed line(s); trace evidence is incomplete`);
+    const finalStatus = qualificationFailure ? "unqualified"
+        : !sawSettled || activeCalls > 0 ? "incomplete"
+            : captureErrors.length > 0 ? "unavailable"
+                : lastStopReason === "error" ? "error"
+                    : lastStopReason === "aborted" ? "aborted"
+                        : lastStopReason === "length" ? "truncated"
+                            : lastStopReason === "stop" && eligibleFinal && currentAssistantText.length > 0 ? "complete" : "unavailable";
+    if (settledContract && finalStatus !== "complete") {
+        const reason = qualificationFailure
+            ?? (finalStatus === "incomplete" ? "current settlement is missing or tool calls remain outstanding"
+                : finalStatus === "truncated" ? "settled assistant reached its output length limit"
+                    : finalStatus === "error" || finalStatus === "aborted" ? `settled assistant stop reason is ${lastStopReason}`
+                        : "current terminal assistant content is empty, ineligible, or captured evidence is malformed");
+        captureErrors.push(`Pi ${meta.piVersion ?? "(version unavailable)"} final delivery is ${finalStatus}; ${reason}`);
+    }
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
@@ -172,7 +252,9 @@ export function parseTrace(lines, meta) {
         // and `smoke-real-pi.sh` asserts no `/home/` survives into a persisted trace
         // — an assertion that used to pass only because the smoke model happened not
         // to echo one.
-        final_text: redactText(finalText || lastAssistantText, meta.homeDir),
+        final_text: redactText(settledContract ? currentAssistantText : finalText || lastAssistantText, meta.homeDir),
+        ...(settledContract ? { final_status: finalStatus } : {}),
+        ...(captureErrors.length ? { capture_errors: [...new Set(captureErrors)] } : {}),
         tool_calls: toolCalls,
         // `null`, not `[]`: the stream says nothing about the filesystem. The runner
         // overwrites this after observing the workspace. Defaulting to `[]` claimed
@@ -184,7 +266,7 @@ export function parseTrace(lines, meta) {
         metrics,
     };
     trace.trace_sha256 = traceSha256(trace);
-    return { trace, isComplete: sawTerminal, malformedLines };
+    return { trace, isComplete: settledContract ? !qualificationFailure && sawSettled && activeCalls === 0 : sawTerminal, malformedLines };
 }
 function addReported(current, value) {
     return typeof value === "number" && value > 0 ? (current ?? 0) + value : current;
@@ -194,8 +276,7 @@ function assistantText(msg) {
     return (msg.content ?? [])
         .filter((b) => b.type === "text" && typeof b.text === "string")
         .map((b) => b.text)
-        .join("\n")
-        .trim();
+        .join("");
 }
 /**
  * Bounded metadata about a tool result. The body is never kept: results carry

package/dist/provider-failure.d.ts

@@ -31,6 +31,10 @@
  * convention cannot drift apart.
  */
 export declare const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/** Failed or unproven terminal delivery, without assuming a provider-side cause. */
+export declare const EXECUTION_FAILURE_MARKER = "[skill-harness] execution failure:";
+export declare function withExecutionFailure(transcript: string, failure: string | null): string;
+export declare function executionFailureFromTranscript(transcript: string): string | null;
 /**
  * Prefix `transcript` with the provider-failure marker, or return it unchanged.
  *

package/dist/provider-failure.js

@@ -31,6 +31,14 @@
  * convention cannot drift apart.
  */
 export const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/** Failed or unproven terminal delivery, without assuming a provider-side cause. */
+export const EXECUTION_FAILURE_MARKER = "[skill-harness] execution failure:";
+export function withExecutionFailure(transcript, failure) {
+    return failure ? `${EXECUTION_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
+}
+export function executionFailureFromTranscript(transcript) {
+    return failureFromPreamble(transcript, EXECUTION_FAILURE_MARKER);
+}
 /**
  * The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),
  * which is where the preamble ends and model-influenced content begins.
@@ -87,11 +95,14 @@ export function providerFailureFromJsonLine(line) {
  * provider failed — it is evidence the model typed the words.
  */
 export function providerFailureFromTranscript(transcript) {
+    return failureFromPreamble(transcript, PROVIDER_FAILURE_MARKER);
+}
+function failureFromPreamble(transcript, marker) {
     for (const line of transcript.split("\n")) {
         if (line.startsWith(TURN_HEADER_PREFIX))
             return null;
-        if (line.startsWith(PROVIDER_FAILURE_MARKER))
-            return line.slice(PROVIDER_FAILURE_MARKER.length).trim();
+        if (line.startsWith(marker))
+            return line.slice(marker.length).trim();
     }
     return null;
 }

package/dist/regrade.js

@@ -5,7 +5,7 @@ import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResult
 import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
-import { providerFailureFromTranscript } from "./provider-failure.js";
+import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -47,8 +47,9 @@ export async function judgeOneRep(opts) {
     // into a model verdict, spending a judge call to turn infrastructure noise into
     // a false FAIL.
     const providerFailure = providerFailureFromTranscript(transcript);
-    if (providerFailure) {
-        const reason = `provider failure — ${providerFailure}`;
+    const executionFailure = executionFailureFromTranscript(transcript);
+    if (providerFailure || executionFailure) {
+        const reason = providerFailure ? `provider failure — ${providerFailure}` : `execution failure — ${executionFailure}`;
         appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
         return {
             verdict: "ERROR", reason, suspect: false,

package/dist/run.js

@@ -16,7 +16,7 @@ import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
-import { providerFailureFromTranscript } from "./provider-failure.js";
+import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
 /**
  * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
  * `PI_GRANTS_LEDGER` at exactly this filename inside the run directory. A fixed
@@ -303,6 +303,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                 // and the artifact disagreeing, and a recovered measurement thrown away.
                 infrastructureFailure = null;
                 adapterFailure = null;
+                let executionUnavailable = false;
                 try {
                     if (scenario.mode === "seeded") {
                         const r = await runSeeded(scenario, {
@@ -348,8 +349,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                             const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
                             transcript = structured.transcript;
                             traces = structured.traces;
+                            executionUnavailable = Boolean(structured.executionFailure);
                             if (structured.providerFailure)
                                 infrastructureFailure = `provider failure — ${structured.providerFailure}`;
+                            else if (structured.executionFailure)
+                                infrastructureFailure = `execution failure — ${structured.executionFailure}`;
                         }
                         else {
                             transcript = await ctx.adapter.run(req);
@@ -384,9 +388,19 @@ async function runRep(scenario, rep, repCount, ctx) {
                     const provider = providerFailureFromTranscript(transcript);
                     if (provider)
                         infrastructureFailure = `provider failure — ${provider}`;
+                    else {
+                        const execution = executionFailureFromTranscript(transcript);
+                        if (execution)
+                            infrastructureFailure = `execution failure — ${execution}`;
+                    }
                 }
-                noResponse = hasEmptyAssistantTurn(transcript);
-                if (!noResponse && !adapterFailure)
+                executionUnavailable ||= executionFailureFromTranscript(transcript) !== null;
+                // Qualified complete traces prove delivered bytes even if those bytes
+                // are only whitespace; the legacy text-only timeout heuristic cannot erase them.
+                const deliveredText = traces.length > 0 && traces.every((trace) => trace.final_status === "complete" && trace.final_text.length > 0 && !trace.capture_errors?.length);
+                noResponse = !deliveredText && hasEmptyAssistantTurn(transcript);
+                // Known unavailable execution is not an empty model answer to retry.
+                if (executionUnavailable || (!noResponse && !adapterFailure))
                     break;
             }
             // Survived the retry: ERROR for this rep, inside a run that still completes
@@ -578,7 +592,7 @@ export function formatScorecard(summary, lift, stability) {
         const tokens = metrics.input_tokens === null
             ? "subject tokens unavailable"
             : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
-        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
+        const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} requested Agent task(s), max outstanding tool calls ${metrics.max_concurrency ?? 0}; child launches/concurrency unobserved`;
         const cost = metrics.cost_source === "subscription"
             ? " · subscription ($0 marginal cost recorded)"
             : metrics.cost_source === "unreported"

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.23.0",
+    "version": "0.24.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",