@skill-harness/adapters 0.23.0 → 0.24.0

package/README.md

@@ -4,6 +4,10 @@ Harness adapters for [skill-harness](https://github.com/mojomanyana/skill-harnes
 the Pi subject/judge runner, the frozen pi-daddy 0.17/v2
 normalizer, and the separately pinned production ledger-v3 normalizer.
 
+Fresh subject execution requires **exactly Pi 1.0.4**. Different or unavailable
+versions produce execution ERROR before a subject call. Retained results remain
+readable; the historical Pi 0.83.0 raw parser is not current qualification.
+
 Internal — use the `skill-harness` CLI. See the repo for docs.
 
 ESM, Node ≥ 20.

package/dist/pi-json.js

@@ -57,8 +57,22 @@ export function runPiJson(opts) {
             // `forbid_calls` gate on that tool then passed for want of the evidence.
             if (!line.trim())
                 return;
-            if (SKIPPED_TYPE_RE.test(line))
+            if (SKIPPED_TYPE_RE.test(line)) {
+                // Validate but never retain the quadratic payload. Otherwise a malformed
+                // update could disappear and make an incomplete capture look successful.
+                try {
+                    const event = JSON.parse(line);
+                    // A late update invalidates an earlier final/settlement. Consecutive
+                    // updates need only one small state marker, not their accumulated text.
+                    if (event.type === "message_update" && kept.at(-1) !== '{"type":"message_update"}') {
+                        kept.push('{"type":"message_update"}');
+                    }
+                }
+                catch {
+                    kept.push("null"); // one parser-owned malformed-record error per bad line
+                }
                 return;
+            }
             kept.push(line);
             if (providerFailure === null)
                 providerFailure = providerFailureFromJsonLine(line);

package/dist/pi.js

@@ -3,7 +3,7 @@ import { tmpdir, homedir } from "node:os";
 import { join, resolve } from "node:path";
 import { runPiJson } from "./pi-json.js";
 import { priceSubjectUsage } from "./model-pricing.js";
-import { exec, onPath, envNum, traceSha256, withProviderFailure } from "@skill-harness/core";
+import { exec, onPath, envNum, piQualificationFailure, withProviderFailure, withExecutionFailure } from "@skill-harness/core";
 const PI_TIMEOUT_MS = envNum("PI_TIMEOUT_MS", 300_000);
 /**
  * stderr fragments that mean the provider refused the request, so the run measured
@@ -107,12 +107,12 @@ export const piAdapter = {
      * results.yaml as `harness_cli_version`.
      *
      * Null on any failure — a non-zero exit, empty output, or pi missing entirely.
-     * A run must not abort because provenance was unavailable, and a fabricated
-     * version would be worse than an absent one.
+     * The probe never fabricates a version. Fresh subject execution refuses an
+     * unavailable or unqualified result before starting the model.
      */
-    async version() {
+    async version(options) {
         try {
-            const r = await exec("pi", ["--version"], { timeoutMs: 30_000 });
+            const r = await exec("pi", ["--version"], { timeoutMs: 30_000, ...options });
             const line = r.stdout.split("\n")[0]?.trim() ?? "";
             if (r.code !== 0 || line === "")
                 return null;
@@ -142,12 +142,15 @@ export const piAdapter = {
         const flags = req.systemPromptFile
             ? ["--no-skills", "--append-system-prompt", readFileSync(req.systemPromptFile, "utf8")]
             : skillFlags(req.mode, req.skillDir);
+        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
+        const qualificationFailure = piQualificationFailure(await this.version({ cwd: req.cwd, env }));
+        if (qualificationFailure)
+            return withExecutionFailure("", qualificationFailure);
         const total = req.turns.length;
         const parts = [];
         // The arm's env, merged over the harness's own — undefined (not `process.env`)
         // when there is none, so `exec`'s `env: opts.env ?? process.env` inherits
         // normally and the control arm is unaffected.
-        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
         // Collected across turns and written by `withProviderFailure` into the
         // transcript PREAMBLE at the end, never inline after an assistant turn: the
         // preamble is the only region of the transcript the model provably cannot
@@ -209,15 +212,23 @@ export const piAdapter = {
         const flags = req.systemPromptFile
             ? ["--no-skills", "--append-system-prompt", readFileSync(req.systemPromptFile, "utf8")]
             : skillFlags(req.mode, req.skillDir);
-        const piVersion = await this.version();
+        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
+        const piVersion = await this.version({ cwd: req.cwd, env });
+        const qualificationFailure = piQualificationFailure(piVersion);
+        if (qualificationFailure)
+            return {
+                transcript: withExecutionFailure("", qualificationFailure),
+                traces: [],
+                executionFailure: qualificationFailure,
+            };
         const total = req.turns.length;
         const traces = [];
         const parts = [];
         const session = total === 1 ? null : mkdtempSync(join(tmpdir(), "sc-pi-session-"));
         let providerFailure = null;
+        let executionFailure = null;
         // Same merge as `run()`: undefined when the arm carries no env, so `spawn`
         // (which treats `undefined` as "inherit") leaves the control arm untouched.
-        const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
         for (let i = 0; i < total; i++) {
             const turnFlags = session === null
                 ? ["--no-session"]
@@ -242,34 +253,44 @@ export const piAdapter = {
             // Fail loudly here rather than let an empty trace satisfy a `forbid_calls`
             // gate — "the model called nothing" and "we recorded nothing" must not
             // reach the scorer looking the same.
-            if (!r.isComplete) {
+            // A settled stream can still end in error/abort, and exit zero is not
+            // terminal success. Preserve the trace and a durable preamble so an
+            // ungated run or later grade cannot turn unavailable delivery into PASS.
+            if (r.trace.final_status !== undefined &&
+                (r.trace.final_status !== "complete" || r.trace.capture_errors?.length || r.code !== 0)) {
+                executionFailure = `Pi ${piVersion} turn ${i + 1}/${total}: final delivery ${r.trace.final_status}` +
+                    ` (exit ${r.code}); ${r.trace.capture_errors?.join("; ") || "successful process completion was not established"}`;
+            }
+            if (!r.isComplete && !executionFailure) {
                 throw new Error(`pi --mode json produced no terminal events for turn ${i + 1}/${total}` +
                     ` (exit ${r.code}${r.malformedLines ? `, ${r.malformedLines} malformed line(s)` : ""})` +
                     (r.stderr.trim() ? `: ${r.stderr.trim()}` : ""));
             }
-            if (r.malformedLines > 0) {
-                r.trace.capture_errors = [`pi JSONL contained ${r.malformedLines} malformed line(s); absence-based trace assertions are unsafe`];
-                r.trace.trace_sha256 = traceSha256(r.trace);
-            }
             // Recorded here, written into the transcript preamble by
             // `withProviderFailure` below — not just returned on `providerFailure`: the
             // artifact on disk is the only thing a later `grade`/`regrade` call ever
             // reads (see `judgeOneRep` in core/regrade.ts), and the structured path
             // exits 0 while carrying the evidence, so a field a re-judge never sees
             // leaves it unrecoverable from the saved transcript.
-            if (providerFailure === null && r.providerFailure)
+            // The qualified settlement contract can prove a successful retry. An
+            // earlier transport diagnostic alone must not override that final result.
+            if (providerFailure === null && r.providerFailure && r.trace.final_status !== "complete") {
                 providerFailure = r.providerFailure;
+            }
             const pricedTrace = priceSubjectUsage(r.trace);
             traces.push(pricedTrace);
             parts.push(header(i + 1, total, req.turns[i]));
-            parts.push(`<<< ASSISTANT:\n${pricedTrace.final_text.trim()}\n`);
+            parts.push(`<<< ASSISTANT:\n${pricedTrace.final_text}\n`);
             if (r.code !== 0)
                 parts.push(`[pi exited ${r.code} on turn ${i + 1}]\n${r.stderr.trim()}\n`);
+            if (executionFailure || providerFailure)
+                break;
         }
         return {
-            transcript: withProviderFailure(parts.join("\n"), providerFailure),
+            transcript: withProviderFailure(withExecutionFailure(parts.join("\n"), executionFailure), providerFailure),
             traces,
             ...(providerFailure ? { providerFailure } : {}),
+            ...(executionFailure ? { executionFailure } : {}),
         };
     },
     /** Run the judge through Pi: no skills, context files, extensions or session. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/adapters",
-    "version": "0.23.0",
+    "version": "0.24.0",
     "description": "skill-harness Pi runner and judge adapter (internal API)",
     "type": "module",
     "license": "MIT",
@@ -42,6 +42,6 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.23.0"
+        "@skill-harness/core": "0.24.0"
     }
 }