@forwardimpact/libeval 0.1.51 → 0.1.52

package/bin/fit-eval.js

@@ -15,19 +15,6 @@ import { runFacilitateCommand } from "../src/commands/facilitate.js";
 import { runDiscussCommand } from "../src/commands/discuss.js";
 import { runCallbackCommand } from "../src/commands/callback.js";
 
-// `tee` streams stdin→stdout via Node's `pipeline`, which needs real stream
-// objects the runtime surface does not expose; it keeps the legacy
-// `(values, args)` signature and this adapter bridges it into dispatch.
-async function teeHandler(ctx) {
-  const out = ctx.args.output;
-  try {
-    await runTeeCommand(ctx.options, out ? [out] : []);
-    return { ok: true };
-  } catch (error) {
-    return { ok: false, code: 1, error: error.message };
-  }
-}
-
 // `bun build --compile` injects FIT_EVAL_VERSION via --define, eliminating
 // the readFileSync branch in the compiled binary (which would ENOENT against
 // the bunfs virtual mount). Source execution falls through to package.json.
@@ -249,7 +236,7 @@ const definition = {
       name: "tee",
       args: ["output"],
       argsUsage: "[output.ndjson]",
-      handler: teeHandler,
+      handler: runTeeCommand,
       description:
         "Stream readable text to stdout while saving raw NDJSON to a file",
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.51",
+  "version": "0.1.52",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/benchmark/apm-installer.js

@@ -4,26 +4,25 @@
  * staging directory, and computes the manifest fingerprint from the lockfile.
  * Per-task copy happens later in WorkdirManager.
  *
- * The class takes a `spawn` seam so tests can substitute a fake child process
- * without ever shelling out to a real `apm` binary. See `createApmInstaller`
- * for the real-dependency wiring; `installApm` is a thin free-function wrapper
- * for callers that don't need to inject anything.
+ * Subprocess and filesystem access route through the injected `runtime` bag
+ * (`runtime.subprocess.spawn` for the streaming `apm` child, `runtime.fs` for
+ * the async staging copies). See `createApmInstaller` for the real-dependency
+ * wiring; `installApm` is a thin free-function wrapper.
  */
 
-import { spawn as nodeSpawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { access, cp, mkdir, readFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 
 /** Installs apm and stages `.claude/` for a task family. */
 export class ApmInstaller {
   /**
-   * @param {object} [deps]
-   * @param {typeof nodeSpawn} [deps.spawn] - Spawn seam (defaults to
-   *   `node:child_process` spawn). Tests inject a fake to avoid shelling out.
+   * @param {object} deps
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `subprocess.spawn` and `fs`.
    */
-  constructor({ spawn } = {}) {
-    this.spawn = spawn ?? nodeSpawn;
+  constructor({ runtime }) {
+    if (!runtime) throw new Error("runtime is required");
+    this.runtime = runtime;
   }
 
   /**
@@ -32,19 +31,21 @@ export class ApmInstaller {
    * @returns {Promise<{stagingDir: string, skillSetHash: string, judgeProfilesDir: string}>}
    */
   async install(family, outputDir) {
+    const fs = this.runtime.fs;
     const stagingDir = join(outputDir, ".apm-staging");
     const stagedClaude = join(stagingDir, ".claude");
     const sourceClaude = join(family.rootPath, ".claude");
     const apmYml = join(family.rootPath, "apm.yml");
 
-    const hasApm = await access(apmYml)
+    const hasApm = await fs
+      .access(apmYml)
       .then(() => true)
       .catch(() => false);
 
     if (hasApm) {
       await this.#runApmInstall(family.rootPath);
       try {
-        await access(sourceClaude);
+        await fs.access(sourceClaude);
       } catch {
         throw new Error(
           `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
@@ -52,14 +53,15 @@ export class ApmInstaller {
       }
     }
 
-    await rm(stagingDir, { recursive: true, force: true });
-    const hasClaudeDir = await access(sourceClaude)
+    await fs.rm(stagingDir, { recursive: true, force: true });
+    const hasClaudeDir = await fs
+      .access(sourceClaude)
       .then(() => true)
       .catch(() => false);
     if (hasClaudeDir) {
-      await cp(sourceClaude, stagedClaude, { recursive: true });
+      await fs.cp(sourceClaude, stagedClaude, { recursive: true });
     } else {
-      await mkdir(stagedClaude, { recursive: true });
+      await fs.mkdir(stagedClaude, { recursive: true });
     }
 
     // Stage the family-local judge profile outside .claude/ so it is available
@@ -67,15 +69,15 @@ export class ApmInstaller {
     const judgeSource = join(family.rootPath, "judge.md");
     const judgeProfilesDir = join(stagingDir, "judge-profiles");
     try {
-      await access(judgeSource);
-      await mkdir(judgeProfilesDir, { recursive: true });
-      await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
+      await fs.access(judgeSource);
+      await fs.mkdir(judgeProfilesDir, { recursive: true });
+      await fs.cp(judgeSource, join(judgeProfilesDir, "judge.md"));
     } catch {}
 
     const lockPath = join(family.rootPath, "apm.lock.yaml");
     let skillSetHash = "";
     try {
-      const lockBytes = await readFile(lockPath);
+      const lockBytes = await fs.readFile(lockPath);
       skillSetHash =
         "sha256:" +
         createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
@@ -86,25 +88,26 @@ export class ApmInstaller {
     return { stagingDir, skillSetHash, judgeProfilesDir };
   }
 
-  #runApmInstall(cwd) {
-    return new Promise((res, rej) => {
-      const child = this.spawn("apm", ["install", "--target", "claude"], {
-        cwd,
-        stdio: ["ignore", "pipe", "pipe"],
-      });
-      let stderr = "";
-      child.stdout.on("data", () => {});
-      child.stderr.on("data", (d) => {
-        stderr += d.toString();
-      });
-      child.on("error", (e) => {
-        rej(new Error(`failed to spawn apm: ${e.message}`));
-      });
-      child.on("close", (code) => {
-        if (code === 0) res();
-        else rej(new Error(`apm install exited ${code}: ${stderr}`));
-      });
-    });
+  async #runApmInstall(cwd) {
+    const child = this.runtime.subprocess.spawn(
+      "apm",
+      ["install", "--target", "claude"],
+      { cwd, stdio: ["ignore", "pipe", "pipe"] },
+    );
+    // Drain stdout concurrently so the child never blocks on backpressure;
+    // capture stderr for the failure message.
+    let stderr = "";
+    const drainStdout = (async () => {
+      for await (const _chunk of child.stdout) {
+        // discard
+      }
+    })();
+    for await (const chunk of child.stderr) stderr += chunk.toString();
+    await drainStdout;
+    const code = await child.exitCode;
+    if (code !== 0) {
+      throw new Error(`apm install exited ${code}: ${stderr}`);
+    }
   }
 }
 
@@ -119,7 +122,7 @@ function normalizeLf(buf) {
 
 /**
  * Factory function — wires real dependencies.
- * @param {ConstructorParameters<typeof ApmInstaller>[0]} [deps]
+ * @param {ConstructorParameters<typeof ApmInstaller>[0]} deps
  * @returns {ApmInstaller}
  */
 export function createApmInstaller(deps) {
@@ -127,10 +130,11 @@ export function createApmInstaller(deps) {
 }
 
 /**
- * Free-function shorthand for callers that don't need to inject a spawn seam.
+ * Free-function shorthand for callers that thread a runtime bag.
  * @param {import("./task-family.js").TaskFamily} family
  * @param {string} outputDir
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  */
-export function installApm(family, outputDir) {
-  return new ApmInstaller().install(family, outputDir);
+export function installApm(family, outputDir, runtime) {
+  return new ApmInstaller({ runtime }).install(family, outputDir);
 }

package/src/benchmark/invariants.js

@@ -2,16 +2,12 @@
  * Invariants — runs `<task.paths.hooks>/invariants.sh` from the template path
  * against the post-run agent CWD. The exit code is authoritative for the
  * verdict; structured per-check rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON.
+ *
+ * Subprocess access flows through `runtime.subprocess.spawn`; the fd-3 backing
+ * store and the stderr log use the sync filesystem surface (`runtime.fsSync`) —
+ * the only surface this module touches, per design Decision 7.
  */
 
-import { spawn } from "node:child_process";
-import {
-  closeSync,
-  createWriteStream,
-  openSync,
-  readFileSync,
-  unlinkSync,
-} from "node:fs";
 import { join } from "node:path";
 
 /**
@@ -25,72 +21,64 @@ import { join } from "node:path";
  * Run the task's invariants script.
  * @param {import("./task-family.js").Task} task
  * @param {{cwd: string, port: number, runDir: string}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<InvariantsResult>}
  */
-export function runInvariants(task, ctx) {
+export async function runInvariants(task, ctx, runtime) {
+  if (!runtime) throw new Error("runtime is required");
   if (!task.paths.invariants) {
-    return Promise.resolve({ verdict: "pass", details: [], exitCode: 0 });
+    return { verdict: "pass", details: [], exitCode: 0 };
   }
-  return new Promise((res, rej) => {
-    const script = task.paths.invariants;
-    const stderrLog = createWriteStream(
-      join(ctx.runDir, "invariants.stderr.log"),
-    );
+  const fsSync = runtime.fsSync;
+  const script = task.paths.invariants;
+  const stderrLogPath = join(ctx.runDir, "invariants.stderr.log");
 
-    // Bun's child_process pipe setup for fd >= 3 is racy under load (it
-    // creates a unix socket pair and the connect() can return ENOENT). Use
-    // a temp file as the fd-3 backing store instead — the script still
-    // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
-    const fd3Path = join(ctx.runDir, "invariants.fd3.ndjson");
-    let fd3File;
-    try {
-      fd3File = openSync(fd3Path, "w+");
-    } catch (e) {
-      rej(e);
-      return;
-    }
+  // Bun's child_process pipe setup for fd >= 3 is racy under load (it
+  // creates a unix socket pair and the connect() can return ENOENT). Use
+  // a temp file as the fd-3 backing store instead — the script still
+  // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
+  const fd3Path = join(ctx.runDir, "invariants.fd3.ndjson");
+  const fd3File = fsSync.openSync(fd3Path, "w+");
 
-    const child = spawn(script, [], {
+  let child;
+  try {
+    child = runtime.subprocess.spawn(script, [], {
       env: {
-        ...process.env,
+        ...runtime.proc.env,
         WORKDIR: ctx.cwd,
         PORT: String(ctx.port),
         RESULTS_FD: "3",
       },
       stdio: ["inherit", "pipe", "pipe", fd3File],
     });
-    if (child.pid === undefined) {
-      try {
-        closeSync(fd3File);
-      } catch {
-        // already closed
-      }
-      rej(new Error(`failed to spawn invariants script: ${script}`));
-      return;
+  } catch (e) {
+    tryClose(fsSync, fd3File);
+    throw e;
+  }
+
+  // Drain stdout (do not require consumers to read it); capture stderr to log.
+  const drainStdout = (async () => {
+    for await (const _chunk of child.stdout) {
+      // discard
     }
+  })();
+  let stderr = "";
+  for await (const chunk of child.stderr) stderr += chunk.toString();
+  await drainStdout;
+  const code = await child.exitCode;
 
-    child.stderr.pipe(stderrLog);
-    // Drain stdout (do not require consumers to read it).
-    child.stdout.on("data", () => {});
+  fsSync.writeFileSync(stderrLogPath, stderr);
+  tryClose(fsSync, fd3File);
 
-    child.on("error", (e) => {
-      tryClose(fd3File);
-      rej(e);
-    });
-    child.on("close", (code) => {
-      stderrLog.end();
-      tryClose(fd3File);
-      const raw = readAndUnlink(fd3Path);
-      const details = [];
-      parseFd3Buffer(raw, details);
-      const exitCode = typeof code === "number" ? code : -1;
-      res({
-        verdict: exitCode === 0 ? "pass" : "fail",
-        details,
-        exitCode,
-      });
-    });
-  });
+  const raw = readAndUnlink(fsSync, fd3Path);
+  const details = [];
+  parseFd3Buffer(raw, details);
+  const exitCode = typeof code === "number" ? code : -1;
+  return {
+    verdict: exitCode === 0 ? "pass" : "fail",
+    details,
+    exitCode,
+  };
 }
 
 function pushRow(line, details) {
@@ -103,23 +91,23 @@ function pushRow(line, details) {
   }
 }
 
-function tryClose(fd) {
+function tryClose(fsSync, fd) {
   try {
-    closeSync(fd);
+    fsSync.closeSync(fd);
   } catch {
     // already closed
   }
 }
 
-function readAndUnlink(path) {
+function readAndUnlink(fsSync, path) {
   let raw = "";
   try {
-    raw = readFileSync(path, "utf8");
+    raw = fsSync.readFileSync(path, "utf8");
   } catch {
     // empty
   }
   try {
-    unlinkSync(path);
+    fsSync.unlinkSync(path);
   } catch {
     // best-effort cleanup
   }

package/src/benchmark/judge.js

@@ -21,9 +21,6 @@
  * historical run from its judge.ndjson file).
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { readFile } from "node:fs/promises";
-import { createInterface } from "node:readline";
 import { createJudge } from "../judge.js";
 import { createRedactor } from "../redaction.js";
 
@@ -45,12 +42,15 @@ import { createRedactor } from "../redaction.js";
  * @param {import("./task-family.js").Task} task
  * @param {import("./workdir.js").Workdir} workdir
  * @param {import("./invariants.js").InvariantsResult} invariants
- * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string}} deps
+ * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string, runtime: import("@forwardimpact/libutil/runtime").Runtime}} deps
  * @param {JudgeContext} [context]
  * @returns {Promise<JudgeVerdict>}
  */
 export async function runJudge(task, workdir, invariants, deps, context) {
-  const template = await readFile(task.paths.judge, "utf8");
+  const runtime = deps.runtime;
+  if (!runtime) throw new Error("runtime is required");
+  const fs = runtime.fs;
+  const template = await fs.readFile(task.paths.judge, "utf8");
   const invariantsJson = JSON.stringify(invariants, null, 2);
   const taskText = template
     .replaceAll("{{INVARIANTS_RESULT}}", invariantsJson)
@@ -61,7 +61,7 @@ export async function runJudge(task, workdir, invariants, deps, context) {
     .replaceAll("{{TASK_ID}}", task.id)
     .replaceAll("{{TASK_DIR}}", workdir.cwd);
 
-  const output = createWriteStream(workdir.judgeTracePath);
+  const output = fs.createWriteStream(workdir.judgeTracePath);
   const judge = createJudge({
     cwd: workdir.cwd,
     query: deps.query,
@@ -70,7 +70,8 @@ export async function runJudge(task, workdir, invariants, deps, context) {
     judgeProfile: deps.judgeProfile,
     profilesDir: deps.profilesDir,
     maxTurns: 25,
-    redactor: createRedactor(),
+    redactor: createRedactor({ runtime }),
+    runtime,
   });
 
   let outcome;
@@ -95,13 +96,14 @@ export async function runJudge(task, workdir, invariants, deps, context) {
  * and map the verdict (`success → pass`, `failure → fail`). Preserved for
  * offline analysis; not used on the runtime happy path.
  * @param {string} tracePath
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<JudgeVerdict | null>}
  */
-export async function parseConcludeFromTrace(tracePath) {
-  const stream = createReadStream(tracePath);
-  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+export async function parseConcludeFromTrace(tracePath, runtime) {
+  if (!runtime) throw new Error("runtime is required");
+  const content = await runtime.fs.readFile(tracePath, "utf8");
   let last = null;
-  for await (const line of rl) {
+  for (const line of content.split("\n")) {
     const candidate = extractConcludeInput(line);
     if (candidate) last = candidate;
   }

package/src/benchmark/npm-installer.js

@@ -3,23 +3,22 @@
  * is present, then copies the resulting `node_modules/` into the staging
  * directory so WorkdirManager can seed each per-task CWD.
  *
- * Symmetric to ApmInstaller: constructor injection of `spawn` for testability,
- * factory function, and a free-function shorthand.
+ * Symmetric to ApmInstaller: the subprocess and filesystem flow through the
+ * injected `runtime` bag (`runtime.subprocess.spawn` + `runtime.fs`).
  */
 
-import { spawn as nodeSpawn } from "node:child_process";
-import { access, cp } from "node:fs/promises";
 import { join } from "node:path";
 
 /** Run `bun install` in the family root and stage node_modules/ for per-task CWDs. */
 export class NpmInstaller {
   /**
-   * @param {object} [deps]
-   * @param {typeof nodeSpawn} [deps.spawn] - Spawn seam (defaults to
-   *   `node:child_process` spawn). Tests inject a fake to avoid shelling out.
+   * @param {object} deps
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `subprocess.spawn` and `fs`.
    */
-  constructor({ spawn } = {}) {
-    this.spawn = spawn ?? nodeSpawn;
+  constructor({ runtime }) {
+    if (!runtime) throw new Error("runtime is required");
+    this.runtime = runtime;
   }
 
   /**
@@ -28,8 +27,10 @@ export class NpmInstaller {
    * @returns {Promise<void>}
    */
   async install(family, stagingDir) {
+    const fs = this.runtime.fs;
     const pkgJson = join(family.rootPath, "package.json");
-    const hasPkg = await access(pkgJson)
+    const hasPkg = await fs
+      .access(pkgJson)
       .then(() => true)
       .catch(() => false);
     if (!hasPkg) return;
@@ -38,37 +39,35 @@ export class NpmInstaller {
 
     const sourceModules = join(family.rootPath, "node_modules");
     try {
-      await access(sourceModules);
+      await fs.access(sourceModules);
     } catch {
       throw new Error(
         `bun install did not produce node_modules/ at ${sourceModules}; check the family's package.json`,
       );
     }
 
-    await cp(sourceModules, join(stagingDir, "node_modules"), {
+    await fs.cp(sourceModules, join(stagingDir, "node_modules"), {
       recursive: true,
     });
   }
 
-  #runBunInstall(cwd) {
-    return new Promise((res, rej) => {
-      const child = this.spawn("bun", ["install"], {
-        cwd,
-        stdio: ["ignore", "pipe", "pipe"],
-      });
-      let stderr = "";
-      child.stdout.on("data", () => {});
-      child.stderr.on("data", (d) => {
-        stderr += d.toString();
-      });
-      child.on("error", (e) => {
-        rej(new Error(`failed to spawn bun: ${e.message}`));
-      });
-      child.on("close", (code) => {
-        if (code === 0) res();
-        else rej(new Error(`bun install exited ${code}: ${stderr}`));
-      });
+  async #runBunInstall(cwd) {
+    const child = this.runtime.subprocess.spawn("bun", ["install"], {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
     });
+    let stderr = "";
+    const drainStdout = (async () => {
+      for await (const _chunk of child.stdout) {
+        // discard
+      }
+    })();
+    for await (const chunk of child.stderr) stderr += chunk.toString();
+    await drainStdout;
+    const code = await child.exitCode;
+    if (code !== 0) {
+      throw new Error(`bun install exited ${code}: ${stderr}`);
+    }
   }
 }
 
@@ -78,10 +77,11 @@ export function createNpmInstaller(deps) {
 }
 
 /**
- * Free-function shorthand for callers that don't need to inject a spawn seam.
+ * Free-function shorthand for callers that thread a runtime bag.
  * @param {import("./task-family.js").TaskFamily} family
  * @param {string} stagingDir
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  */
-export function installNpm(family, stagingDir) {
-  return new NpmInstaller().install(family, stagingDir);
+export function installNpm(family, stagingDir, runtime) {
+  return new NpmInstaller({ runtime }).install(family, stagingDir);
 }

package/src/benchmark/report.js

@@ -12,9 +12,7 @@
  * whole report.
  */
 
-import { createReadStream } from "node:fs";
 import { join } from "node:path";
-import { createInterface } from "node:readline";
 
 import { validateResultRecord } from "./result.js";
 
@@ -41,11 +39,17 @@ import { validateResultRecord } from "./result.js";
  */
 
 /**
- * @param {{inputDir: string, kValues: number[], includeRuns?: boolean}} opts
+ * @param {{inputDir: string, kValues: number[], includeRuns?: boolean, runtime: import("@forwardimpact/libutil/runtime").Runtime}} opts
  * @returns {Promise<{tasks: TaskReport[], totals: object}>}
  */
-export async function aggregate({ inputDir, kValues, includeRuns = false }) {
-  const records = await loadRecords(inputDir);
+export async function aggregate({
+  inputDir,
+  kValues,
+  includeRuns = false,
+  runtime,
+}) {
+  if (!runtime) throw new Error("runtime is required");
+  const records = await loadRecords(inputDir, runtime);
   const grouped = groupByTask(records.records);
   const tasks = [];
   let totalRuns = 0;
@@ -429,20 +433,30 @@ function median(arr) {
 // Record loading
 // ---------------------------------------------------------------------------
 
-async function loadRecords(inputDir) {
+async function loadRecords(inputDir, runtime) {
   const path = join(inputDir, "results.jsonl");
-  const stream = createReadStream(path);
-  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  let content;
+  try {
+    content = await runtime.fs.readFile(path, "utf8");
+  } catch (e) {
+    // Re-throw with the stack collapsed to the message line so the CLI's
+    // error rendering stays free of node-internal async `readFile` frames
+    // (matching the pre-1370 stream-error shape the golden captured).
+    const err = new Error(e.message);
+    if (e.code) err.code = e.code;
+    err.stack = `Error: ${e.message}`;
+    throw err;
+  }
   const records = [];
   let skipped = 0;
-  for await (const line of rl) {
+  for (const line of content.split("\n")) {
     const trimmed = line.trim();
     if (!trimmed) continue;
     let record;
     try {
       record = JSON.parse(trimmed);
     } catch (e) {
-      process.stderr.write(
+      runtime.proc.stderr.write(
         `benchmark report: skipped malformed JSON line — ${e.message}\n`,
       );
       skipped++;
@@ -451,7 +465,7 @@ async function loadRecords(inputDir) {
     try {
       validateResultRecord(record);
     } catch (e) {
-      process.stderr.write(
+      runtime.proc.stderr.write(
         `benchmark report: skipped record failing schema — ${describeError(e)}\n`,
       );
       skipped++;