@skill-harness/core 0.25.0 → 0.26.0

package/dist/provider-failure.js

@@ -37,7 +37,18 @@ export function withExecutionFailure(transcript, failure) {
     return failure ? `${EXECUTION_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
 }
 export function executionFailureFromTranscript(transcript) {
-    return failureFromPreamble(transcript, EXECUTION_FAILURE_MARKER);
+    const failure = failureFromPreamble(transcript, EXECUTION_FAILURE_MARKER);
+    if (failure !== null)
+        return failure;
+    // Before 0.25.1, the runner saved thrown errors as this entire transcript,
+    // without a turn header. Match only its exact opening preamble: a model quote
+    // after a USER header (or elsewhere in the transcript) carries no authority.
+    const legacyPrefix = "[adapter failure] ";
+    if (transcript.startsWith(legacyPrefix)) {
+        const detail = transcript.slice(legacyPrefix.length).split("\n", 1)[0].trim();
+        return `adapter failure — ${detail || "adapter threw without a message"}`;
+    }
+    return null;
 }
 /**
  * The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),

package/dist/run.js

@@ -16,7 +16,7 @@ import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
-import { providerFailureFromTranscript, executionFailureFromTranscript } from "./provider-failure.js";
+import { providerFailureFromTranscript, executionFailureFromTranscript, withExecutionFailure } from "./provider-failure.js";
 /**
  * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
  * `PI_GRANTS_LEDGER` at exactly this filename inside the run directory. A fixed
@@ -372,7 +372,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                     // Configuration errors do not arrive here: the two that used to be
                     // thrown from inside this block (adapter cannot produce traces for a
                     // gated scenario) are decided above, before the loop.
-                    adapterFailure = e instanceof Error ? e.message : String(e);
+                    adapterFailure = (e instanceof Error ? e.message : String(e)) || "adapter threw without a message";
                     transcript = `[adapter failure] ${adapterFailure}`;
                     // Nothing partial from the failed attempt may be read as evidence: an
                     // empty trace list satisfies a `forbid_calls` gate, and a stale diff
@@ -384,7 +384,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                 // A provider outage is infrastructure, never a model verdict. Checked on
                 // every path: the text path carries the marker in the transcript, the
                 // structured path sets `providerFailure` above and exits 0 while doing it.
-                if (!infrastructureFailure) {
+                if (!infrastructureFailure && adapterFailure === null) {
                     const provider = providerFailureFromTranscript(transcript);
                     if (provider)
                         infrastructureFailure = `provider failure — ${provider}`;
@@ -394,7 +394,8 @@ async function runRep(scenario, rep, repCount, ctx) {
                             infrastructureFailure = `execution failure — ${execution}`;
                     }
                 }
-                executionUnavailable ||= executionFailureFromTranscript(transcript) !== null;
+                // Thrown errors keep their one fresh retry, unlike a returned unavailable final.
+                executionUnavailable ||= adapterFailure === null && executionFailureFromTranscript(transcript) !== null;
                 // Qualified complete traces prove delivered bytes even if those bytes
                 // are only whitespace; the legacy text-only timeout heuristic cannot erase them.
                 const deliveredText = traces.length > 0 && traces.every((trace) => trace.final_status === "complete" && trace.final_text.length > 0 && !trace.capture_errors?.length);
@@ -405,8 +406,11 @@ async function runRep(scenario, rep, repCount, ctx) {
             }
             // Survived the retry: ERROR for this rep, inside a run that still completes
             // and still writes every other scenario's verdict.
-            if (adapterFailure && !infrastructureFailure) {
-                infrastructureFailure = `adapter failure — ${adapterFailure}`;
+            if (adapterFailure) {
+                infrastructureFailure ??= `adapter failure — ${adapterFailure}`;
+                // Persist only the exhausted failure. A recovered retry must retain its
+                // clean transcript; later grade/regate must never judge an unavailable run.
+                transcript = withExecutionFailure(transcript, `adapter failure — ${adapterFailure}`);
             }
         }
         const repSuffix = repCount > 1 ? rep : undefined;

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.25.0",
-    "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
+    "version": "0.26.0",
+    "description": "skill-harness engine \u2014 spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",
     "main": "./dist/index.js",