@skill-harness/adapters 0.10.0 → 0.11.0

package/dist/pi-json.d.ts

@@ -32,6 +32,11 @@ export interface PiJsonRunOptions {
     turn: number;
     changedPaths?: string[];
     homeDir?: string;
+    /**
+     * Extra env for the subject process, from the arm. `spawn` treats `undefined`
+     * as "inherit", so the control arm (which never sets this) is unaffected.
+     */
+    env?: NodeJS.ProcessEnv;
 }
 export interface PiJsonRunResult {
     trace: ExecutionTraceV1;
@@ -39,5 +44,12 @@ export interface PiJsonRunResult {
     malformedLines: number;
     code: number | null;
     stderr: string;
+    /**
+     * Set when a line on the stream carried a provider-side failure diagnostic
+     * (auth, transport) rather than the model answering badly. `runStructured`
+     * collects this across turns; `run.ts` turns it into ERROR — never a model
+     * verdict.
+     */
+    providerFailure: string | null;
 }
 export declare function runPiJson(opts: PiJsonRunOptions): Promise<PiJsonRunResult>;

package/dist/pi-json.js

@@ -1,6 +1,6 @@
 import { spawn } from "node:child_process";
 import { createInterface } from "node:readline";
-import { parseTrace } from "@skill-harness/core";
+import { parseTrace, providerFailureFromJsonLine } from "@skill-harness/core";
 /**
  * Run `pi --mode json` and build an execution trace, **streaming**.
  *
@@ -28,12 +28,14 @@ export function runPiJson(opts) {
     return new Promise((resolve, reject) => {
         const child = spawn("pi", opts.args, {
             cwd: opts.cwd,
+            env: opts.env,
             // stdin from /dev/null: pi hangs waiting on it otherwise, and a hang in a
             // wave is indistinguishable from a slow model until the timeout fires.
             stdio: ["ignore", "pipe", "pipe"],
         });
         const kept = [];
         let stderr = "";
+        let providerFailure = null;
         let settled = false;
         const timer = setTimeout(() => {
             if (settled)
@@ -58,6 +60,8 @@ export function runPiJson(opts) {
             if (SKIPPED_TYPE_RE.test(line))
                 return;
             kept.push(line);
+            if (providerFailure === null)
+                providerFailure = providerFailureFromJsonLine(line);
         });
         child.stderr.on("data", (chunk) => {
             if (stderr.length < MAX_STDERR_CHARS)
@@ -85,7 +89,7 @@ export function runPiJson(opts) {
                 changedPaths: opts.changedPaths,
                 homeDir: opts.homeDir,
             });
-            resolve({ ...parsed, code, stderr: stderr.slice(0, MAX_STDERR_CHARS) });
+            resolve({ ...parsed, code, stderr: stderr.slice(0, MAX_STDERR_CHARS), providerFailure });
         });
     });
 }

package/dist/pi.js

@@ -3,8 +3,24 @@ import { tmpdir, homedir } from "node:os";
 import { join, resolve } from "node:path";
 import { runPiJson } from "./pi-json.js";
 import { collectTrajectorySources, normalizePiTraces, resequence } from "./trajectory.js";
-import { exec, onPath, envNum, traceSha256 } from "@skill-harness/core";
+import { exec, onPath, envNum, traceSha256, withProviderFailure } from "@skill-harness/core";
 const PI_TIMEOUT_MS = envNum("PI_TIMEOUT_MS", 300_000);
+/**
+ * stderr fragments that mean the provider refused the request, so the run measured
+ * nothing about the model. Substring matching on a message pi passes through from
+ * the provider — deliberately narrow: a stderr line we cannot classify stays an
+ * ordinary non-zero exit, because calling a real model failure "infrastructure"
+ * would hide a regression.
+ */
+const PROVIDER_STDERR_SIGNATURES = [
+    "invalidated oauth token",
+    "invalid_api_key",
+    "insufficient_quota",
+];
+function providerStderr(stderr) {
+    const hay = stderr.toLowerCase();
+    return PROVIDER_STDERR_SIGNATURES.some((sig) => hay.includes(sig)) ? stderr.trim() : null;
+}
 /**
  * Refuse to hand pi a skill dir it will silently ignore.
  *
@@ -127,26 +143,46 @@ export const piAdapter = {
             : skillFlags(req.mode, req.skillDir);
         const total = req.turns.length;
         const parts = [];
+        // The arm's env, merged over the harness's own — undefined (not `process.env`)
+        // when there is none, so `exec`'s `env: opts.env ?? process.env` inherits
+        // normally and the control arm is unaffected.
+        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
+        // Collected across turns and written by `withProviderFailure` into the
+        // transcript PREAMBLE at the end, never inline after an assistant turn: the
+        // preamble is the only region of the transcript the model provably cannot
+        // reach, and a marker anywhere else is forgeable by a model that types the
+        // words (which would convert a FAIL into ERROR and mute the judge forever).
+        let providerFailure = null;
         if (total === 1) {
             const args = [...flags, ...common, "--no-session", "-p", req.turns[0]];
-            const r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS });
+            const r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS, env });
             parts.push(header(1, 1, req.turns[0]));
             parts.push(`<<< ASSISTANT:\n${r.stdout.trim()}\n`);
-            if (r.code !== 0)
-                parts.push(`[pi exited ${r.code}]\n${r.stderr.trim()}\n`);
-            return parts.join("\n");
+            if (r.code !== 0) {
+                providerFailure = providerStderr(r.stderr);
+                if (!providerFailure)
+                    parts.push(`[pi exited ${r.code}]\n${r.stderr.trim()}\n`);
+            }
+            return withProviderFailure(parts.join("\n"), providerFailure);
         }
         const session = mkdtempSync(join(tmpdir(), "sc-pi-session-"));
         for (let i = 0; i < total; i++) {
             const turnFlags = i === 0 ? ["--session-dir", session] : ["--session-dir", session, "-c"];
             const args = [...flags, ...common, ...turnFlags, "-p", req.turns[i]];
-            const r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS });
+            const r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS, env });
             parts.push(header(i + 1, total, req.turns[i]));
             parts.push(`<<< ASSISTANT:\n${r.stdout.trim()}\n`);
-            if (r.code !== 0)
-                parts.push(`[pi exited ${r.code} on turn ${i + 1}]\n${r.stderr.trim()}\n`);
+            if (r.code !== 0) {
+                const provider = providerStderr(r.stderr);
+                // First failure wins, same as the structured path: the turns after an
+                // outage are downstream of it, not independent evidence.
+                if (provider && providerFailure === null)
+                    providerFailure = provider;
+                if (!provider)
+                    parts.push(`[pi exited ${r.code} on turn ${i + 1}]\n${r.stderr.trim()}\n`);
+            }
         }
-        return parts.join("\n");
+        return withProviderFailure(parts.join("\n"), providerFailure);
     },
     /**
      * Structured run: same flags, same turn loop, plus `--mode json` and a trace
@@ -178,6 +214,10 @@ export const piAdapter = {
         const traces = [];
         const parts = [];
         const session = total === 1 ? null : mkdtempSync(join(tmpdir(), "sc-pi-session-"));
+        let providerFailure = null;
+        // Same merge as `run()`: undefined when the arm carries no env, so `spawn`
+        // (which treats `undefined` as "inherit") leaves the control arm untouched.
+        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
         for (let i = 0; i < total; i++) {
             const turnFlags = session === null
                 ? ["--no-session"]
@@ -196,6 +236,7 @@ export const piAdapter = {
                 rep: req.rep ?? 0,
                 turn: i,
                 homeDir: homedir(),
+                env,
             });
             // A stream with no terminal events at all is not evidence of a clean run.
             // Fail loudly here rather than let an empty trace satisfy a `forbid_calls`
@@ -210,6 +251,14 @@ export const piAdapter = {
                 r.trace.capture_errors = [`pi JSONL contained ${r.malformedLines} malformed line(s); absence-based trace assertions are unsafe`];
                 r.trace.trace_sha256 = traceSha256(r.trace);
             }
+            // Recorded here, written into the transcript preamble by
+            // `withProviderFailure` below — not just returned on `providerFailure`: the
+            // artifact on disk is the only thing a later `grade`/`regrade` call ever
+            // reads (see `judgeOneRep` in core/regrade.ts), and the structured path
+            // exits 0 while carrying the evidence, so a field a re-judge never sees
+            // leaves it unrecoverable from the saved transcript.
+            if (providerFailure === null && r.providerFailure)
+                providerFailure = r.providerFailure;
             traces.push(r.trace);
             parts.push(header(i + 1, total, req.turns[i]));
             parts.push(`<<< ASSISTANT:\n${r.trace.final_text.trim()}\n`);
@@ -235,10 +284,11 @@ export const piAdapter = {
         }
         const eventErrors = [...native.errors, ...chronologyErrors];
         return {
-            transcript: parts.join("\n"),
+            transcript: withProviderFailure(parts.join("\n"), providerFailure),
             traces,
             events: resequence(combined),
             ...(eventErrors.length ? { eventErrors } : {}),
+            ...(providerFailure ? { providerFailure } : {}),
         };
     },
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/adapters",
-    "version": "0.10.0",
+    "version": "0.11.0",
     "description": "skill-harness harness adapters — pi runner + claude-code judge routing (internal API)",
     "type": "module",
     "license": "MIT",
@@ -40,6 +40,6 @@
         "prepack": "cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.10.0"
+        "@skill-harness/core": "0.11.0"
     }
 }