@forwardimpact/libeval 0.1.51 → 0.1.53

package/bin/fit-benchmark.js

@@ -2,7 +2,7 @@
 
 import "@forwardimpact/libpreflight/node22";
 
-import { readFileSync, realpathSync } from "node:fs";
+import { realpathSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
 import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
@@ -11,17 +11,8 @@ import { runBenchmarkRunCommand } from "../src/commands/benchmark-run.js";
 import { runBenchmarkInvariantsCommand } from "../src/commands/benchmark-invariants.js";
 import { runBenchmarkReportCommand } from "../src/commands/benchmark-report.js";
 
-// `bun build --compile` injects FIT_BENCHMARK_VERSION via --define, eliminating
-// the readFileSync branch in the compiled binary (which would ENOENT against
-// the bunfs virtual mount). Source execution falls through to package.json.
-const VERSION =
-  process.env.FIT_BENCHMARK_VERSION ||
-  JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"))
-    .version;
-
 export const definition = {
   name: "fit-benchmark",
-  version: VERSION,
   description:
     "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
   commands: [
@@ -156,11 +147,14 @@ export const definition = {
   ],
 };
 
-const logger = createLogger("benchmark");
+const runtime = createDefaultRuntime();
+const logger = createLogger("benchmark", runtime);
 
 async function main() {
-  const runtime = createDefaultRuntime();
-  const cli = createCli(definition, { runtime });
+  const cli = createCli(definition, {
+    runtime,
+    packageJsonUrl: new URL("../package.json", import.meta.url),
+  });
   const parsed = cli.parse(runtime.proc.argv.slice(2));
   if (!parsed) return runtime.proc.exit(0);
 
@@ -187,7 +181,7 @@ async function main() {
 if (import.meta.url === `file://${realpathSync(process.argv[1])}`) {
   main().catch((error) => {
     logger.exception("main", error);
-    createCli(definition).error(error.message);
+    createCli(definition, { runtime }).error(error.message);
     process.exit(1);
   });
 }

package/bin/fit-eval.js

@@ -2,7 +2,6 @@
 
 import "@forwardimpact/libpreflight/node22";
 
-import { readFileSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
 import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
@@ -15,27 +14,6 @@ import { runFacilitateCommand } from "../src/commands/facilitate.js";
 import { runDiscussCommand } from "../src/commands/discuss.js";
 import { runCallbackCommand } from "../src/commands/callback.js";
 
-// `tee` streams stdin→stdout via Node's `pipeline`, which needs real stream
-// objects the runtime surface does not expose; it keeps the legacy
-// `(values, args)` signature and this adapter bridges it into dispatch.
-async function teeHandler(ctx) {
-  const out = ctx.args.output;
-  try {
-    await runTeeCommand(ctx.options, out ? [out] : []);
-    return { ok: true };
-  } catch (error) {
-    return { ok: false, code: 1, error: error.message };
-  }
-}
-
-// `bun build --compile` injects FIT_EVAL_VERSION via --define, eliminating
-// the readFileSync branch in the compiled binary (which would ENOENT against
-// the bunfs virtual mount). Source execution falls through to package.json.
-const VERSION =
-  process.env.FIT_EVAL_VERSION ||
-  JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"))
-    .version;
-
 const LEAD_OPTIONS = {
   "lead-profile": {
     type: "string",
@@ -73,7 +51,6 @@ const TASK_INPUT_OPTIONS = {
 
 const definition = {
   name: "fit-eval",
-  version: VERSION,
   description:
     "Run agents and capture NDJSON traces — for agent evaluations or multi-agent collaboration",
   commands: [
@@ -249,7 +226,7 @@ const definition = {
       name: "tee",
       args: ["output"],
       argsUsage: "[output.ndjson]",
-      handler: teeHandler,
+      handler: runTeeCommand,
       description:
         "Stream readable text to stdout while saving raw NDJSON to a file",
     },
@@ -326,11 +303,14 @@ const definition = {
   ],
 };
 
-const logger = createLogger("eval");
+const runtime = createDefaultRuntime();
+const logger = createLogger("eval", runtime);
 
 async function main() {
-  const runtime = createDefaultRuntime();
-  const cli = createCli(definition, { runtime });
+  const cli = createCli(definition, {
+    runtime,
+    packageJsonUrl: new URL("../package.json", import.meta.url),
+  });
   const parsed = cli.parse(runtime.proc.argv.slice(2));
   if (!parsed) return runtime.proc.exit(0);
 
@@ -354,6 +334,6 @@ async function main() {
 
 main().catch((error) => {
   logger.exception("main", error);
-  createCli(definition).error(error.message);
+  createCli(definition, { runtime }).error(error.message);
   process.exit(1);
 });

package/bin/fit-selfedit.js

@@ -7,12 +7,11 @@
 
 import "@forwardimpact/libpreflight/node22";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import fsPromises from "node:fs/promises";
 import { parseArgs } from "node:util";
 import { resolve, relative, dirname } from "node:path";
 import { execFileSync } from "node:child_process";
 
-import { Finder } from "@forwardimpact/libutil";
+import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { minimatch } from "minimatch";
 
 const HELP = `fit-selfedit — write stdin to a settings.json-allowed path on a non-main branch.
@@ -71,8 +70,11 @@ if (extra.length > 0) fail(`unexpected extra arguments: ${extra.join(" ")}`);
 
 const absoluteTarget = resolve(process.cwd(), targetArg);
 
-// Safeguard 1: settings.json must grant Edit() on this path.
-const settingsPath = new Finder(fsPromises, { debug() {} }).findUpward(
+// Safeguard 1: settings.json must grant Edit() on this path. The bin is the
+// sole construction site for the runtime; resolve the finder off the bag
+// rather than constructing a Finder here (Success Criterion 9).
+const runtime = createDefaultRuntime();
+const settingsPath = runtime.finder.findUpward(
   dirname(absoluteTarget),
   ".claude/settings.json",
   20,

package/bin/fit-trace.js

@@ -2,7 +2,6 @@
 
 import "@forwardimpact/libpreflight/node22";
 
-import { readFileSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
 import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createScriptConfig } from "@forwardimpact/libconfig";
@@ -31,17 +30,8 @@ import {
 import { runAssertCommand } from "../src/commands/assert.js";
 import { runByDiscussionCommand } from "../src/commands/by-discussion.js";
 
-// `bun build --compile` injects FIT_TRACE_VERSION via --define, eliminating
-// the readFileSync branch in the compiled binary (which would ENOENT against
-// the bunfs virtual mount). Source execution falls through to package.json.
-const VERSION =
-  process.env.FIT_TRACE_VERSION ||
-  JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"))
-    .version;
-
 const definition = {
   name: "fit-trace",
-  version: VERSION,
   description:
     "Download, query, and analyze agent execution traces — read NDJSON output from fit-eval as qualitative research",
   commands: [
@@ -340,15 +330,18 @@ const definition = {
   ],
 };
 
-const logger = createLogger("trace");
+const runtime = createDefaultRuntime();
+const logger = createLogger("trace", runtime);
 
 // Commands that talk to the GitHub API need a config-backed token resolver;
 // the rest only read local trace files through the runtime.
 const NEEDS_CONFIG = new Set(["runs", "download"]);
 
 async function main() {
-  const runtime = createDefaultRuntime();
-  const cli = createCli(definition, { runtime });
+  const cli = createCli(definition, {
+    runtime,
+    packageJsonUrl: new URL("../package.json", import.meta.url),
+  });
   const parsed = cli.parse(runtime.proc.argv.slice(2));
   if (!parsed) return runtime.proc.exit(0);
 
@@ -376,6 +369,6 @@ async function main() {
 
 main().catch((error) => {
   logger.exception("main", error);
-  createCli(definition).error(error.message);
+  createCli(definition, { runtime }).error(error.message);
   process.exit(1);
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.51",
+  "version": "0.1.53",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/benchmark/apm-installer.js

@@ -4,26 +4,25 @@
  * staging directory, and computes the manifest fingerprint from the lockfile.
  * Per-task copy happens later in WorkdirManager.
  *
- * The class takes a `spawn` seam so tests can substitute a fake child process
- * without ever shelling out to a real `apm` binary. See `createApmInstaller`
- * for the real-dependency wiring; `installApm` is a thin free-function wrapper
- * for callers that don't need to inject anything.
+ * Subprocess and filesystem access route through the injected `runtime` bag
+ * (`runtime.subprocess.spawn` for the streaming `apm` child, `runtime.fs` for
+ * the async staging copies). See `createApmInstaller` for the real-dependency
+ * wiring; `installApm` is a thin free-function wrapper.
  */
 
-import { spawn as nodeSpawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { access, cp, mkdir, readFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 
 /** Installs apm and stages `.claude/` for a task family. */
 export class ApmInstaller {
   /**
-   * @param {object} [deps]
-   * @param {typeof nodeSpawn} [deps.spawn] - Spawn seam (defaults to
-   *   `node:child_process` spawn). Tests inject a fake to avoid shelling out.
+   * @param {object} deps
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `subprocess.spawn` and `fs`.
    */
-  constructor({ spawn } = {}) {
-    this.spawn = spawn ?? nodeSpawn;
+  constructor({ runtime }) {
+    if (!runtime) throw new Error("runtime is required");
+    this.runtime = runtime;
   }
 
   /**
@@ -32,19 +31,21 @@ export class ApmInstaller {
    * @returns {Promise<{stagingDir: string, skillSetHash: string, judgeProfilesDir: string}>}
    */
   async install(family, outputDir) {
+    const fs = this.runtime.fs;
     const stagingDir = join(outputDir, ".apm-staging");
     const stagedClaude = join(stagingDir, ".claude");
     const sourceClaude = join(family.rootPath, ".claude");
     const apmYml = join(family.rootPath, "apm.yml");
 
-    const hasApm = await access(apmYml)
+    const hasApm = await fs
+      .access(apmYml)
       .then(() => true)
       .catch(() => false);
 
     if (hasApm) {
       await this.#runApmInstall(family.rootPath);
       try {
-        await access(sourceClaude);
+        await fs.access(sourceClaude);
       } catch {
         throw new Error(
           `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
@@ -52,14 +53,15 @@ export class ApmInstaller {
       }
     }
 
-    await rm(stagingDir, { recursive: true, force: true });
-    const hasClaudeDir = await access(sourceClaude)
+    await fs.rm(stagingDir, { recursive: true, force: true });
+    const hasClaudeDir = await fs
+      .access(sourceClaude)
       .then(() => true)
       .catch(() => false);
     if (hasClaudeDir) {
-      await cp(sourceClaude, stagedClaude, { recursive: true });
+      await fs.cp(sourceClaude, stagedClaude, { recursive: true });
     } else {
-      await mkdir(stagedClaude, { recursive: true });
+      await fs.mkdir(stagedClaude, { recursive: true });
     }
 
     // Stage the family-local judge profile outside .claude/ so it is available
@@ -67,15 +69,15 @@ export class ApmInstaller {
     const judgeSource = join(family.rootPath, "judge.md");
     const judgeProfilesDir = join(stagingDir, "judge-profiles");
     try {
-      await access(judgeSource);
-      await mkdir(judgeProfilesDir, { recursive: true });
-      await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
+      await fs.access(judgeSource);
+      await fs.mkdir(judgeProfilesDir, { recursive: true });
+      await fs.cp(judgeSource, join(judgeProfilesDir, "judge.md"));
     } catch {}
 
     const lockPath = join(family.rootPath, "apm.lock.yaml");
     let skillSetHash = "";
     try {
-      const lockBytes = await readFile(lockPath);
+      const lockBytes = await fs.readFile(lockPath);
       skillSetHash =
         "sha256:" +
         createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
@@ -86,25 +88,26 @@ export class ApmInstaller {
     return { stagingDir, skillSetHash, judgeProfilesDir };
   }
 
-  #runApmInstall(cwd) {
-    return new Promise((res, rej) => {
-      const child = this.spawn("apm", ["install", "--target", "claude"], {
-        cwd,
-        stdio: ["ignore", "pipe", "pipe"],
-      });
-      let stderr = "";
-      child.stdout.on("data", () => {});
-      child.stderr.on("data", (d) => {
-        stderr += d.toString();
-      });
-      child.on("error", (e) => {
-        rej(new Error(`failed to spawn apm: ${e.message}`));
-      });
-      child.on("close", (code) => {
-        if (code === 0) res();
-        else rej(new Error(`apm install exited ${code}: ${stderr}`));
-      });
-    });
+  async #runApmInstall(cwd) {
+    const child = this.runtime.subprocess.spawn(
+      "apm",
+      ["install", "--target", "claude"],
+      { cwd, stdio: ["ignore", "pipe", "pipe"] },
+    );
+    // Drain stdout concurrently so the child never blocks on backpressure;
+    // capture stderr for the failure message.
+    let stderr = "";
+    const drainStdout = (async () => {
+      for await (const _chunk of child.stdout) {
+        // discard
+      }
+    })();
+    for await (const chunk of child.stderr) stderr += chunk.toString();
+    await drainStdout;
+    const code = await child.exitCode;
+    if (code !== 0) {
+      throw new Error(`apm install exited ${code}: ${stderr}`);
+    }
   }
 }
 
@@ -119,7 +122,7 @@ function normalizeLf(buf) {
 
 /**
  * Factory function — wires real dependencies.
- * @param {ConstructorParameters<typeof ApmInstaller>[0]} [deps]
+ * @param {ConstructorParameters<typeof ApmInstaller>[0]} deps
  * @returns {ApmInstaller}
  */
 export function createApmInstaller(deps) {
@@ -127,10 +130,11 @@ export function createApmInstaller(deps) {
 }
 
 /**
- * Free-function shorthand for callers that don't need to inject a spawn seam.
+ * Free-function shorthand for callers that thread a runtime bag.
  * @param {import("./task-family.js").TaskFamily} family
  * @param {string} outputDir
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  */
-export function installApm(family, outputDir) {
-  return new ApmInstaller().install(family, outputDir);
+export function installApm(family, outputDir, runtime) {
+  return new ApmInstaller({ runtime }).install(family, outputDir);
 }

package/src/benchmark/invariants.js

@@ -2,16 +2,12 @@
  * Invariants — runs `<task.paths.hooks>/invariants.sh` from the template path
  * against the post-run agent CWD. The exit code is authoritative for the
  * verdict; structured per-check rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON.
+ *
+ * Subprocess access flows through `runtime.subprocess.spawn`; the fd-3 backing
+ * store and the stderr log use the sync filesystem surface (`runtime.fsSync`) —
+ * the only surface this module touches, per design Decision 7.
  */
 
-import { spawn } from "node:child_process";
-import {
-  closeSync,
-  createWriteStream,
-  openSync,
-  readFileSync,
-  unlinkSync,
-} from "node:fs";
 import { join } from "node:path";
 
 /**
@@ -25,72 +21,64 @@ import { join } from "node:path";
  * Run the task's invariants script.
  * @param {import("./task-family.js").Task} task
  * @param {{cwd: string, port: number, runDir: string}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<InvariantsResult>}
  */
-export function runInvariants(task, ctx) {
+export async function runInvariants(task, ctx, runtime) {
+  if (!runtime) throw new Error("runtime is required");
   if (!task.paths.invariants) {
-    return Promise.resolve({ verdict: "pass", details: [], exitCode: 0 });
+    return { verdict: "pass", details: [], exitCode: 0 };
   }
-  return new Promise((res, rej) => {
-    const script = task.paths.invariants;
-    const stderrLog = createWriteStream(
-      join(ctx.runDir, "invariants.stderr.log"),
-    );
+  const fsSync = runtime.fsSync;
+  const script = task.paths.invariants;
+  const stderrLogPath = join(ctx.runDir, "invariants.stderr.log");
 
-    // Bun's child_process pipe setup for fd >= 3 is racy under load (it
-    // creates a unix socket pair and the connect() can return ENOENT). Use
-    // a temp file as the fd-3 backing store instead — the script still
-    // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
-    const fd3Path = join(ctx.runDir, "invariants.fd3.ndjson");
-    let fd3File;
-    try {
-      fd3File = openSync(fd3Path, "w+");
-    } catch (e) {
-      rej(e);
-      return;
-    }
+  // Bun's child_process pipe setup for fd >= 3 is racy under load (it
+  // creates a unix socket pair and the connect() can return ENOENT). Use
+  // a temp file as the fd-3 backing store instead — the script still
+  // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
+  const fd3Path = join(ctx.runDir, "invariants.fd3.ndjson");
+  const fd3File = fsSync.openSync(fd3Path, "w+");
 
-    const child = spawn(script, [], {
+  let child;
+  try {
+    child = runtime.subprocess.spawn(script, [], {
       env: {
-        ...process.env,
+        ...runtime.proc.env,
         WORKDIR: ctx.cwd,
         PORT: String(ctx.port),
         RESULTS_FD: "3",
       },
       stdio: ["inherit", "pipe", "pipe", fd3File],
     });
-    if (child.pid === undefined) {
-      try {
-        closeSync(fd3File);
-      } catch {
-        // already closed
-      }
-      rej(new Error(`failed to spawn invariants script: ${script}`));
-      return;
+  } catch (e) {
+    tryClose(fsSync, fd3File);
+    throw e;
+  }
+
+  // Drain stdout (do not require consumers to read it); capture stderr to log.
+  const drainStdout = (async () => {
+    for await (const _chunk of child.stdout) {
+      // discard
     }
+  })();
+  let stderr = "";
+  for await (const chunk of child.stderr) stderr += chunk.toString();
+  await drainStdout;
+  const code = await child.exitCode;
 
-    child.stderr.pipe(stderrLog);
-    // Drain stdout (do not require consumers to read it).
-    child.stdout.on("data", () => {});
+  fsSync.writeFileSync(stderrLogPath, stderr);
+  tryClose(fsSync, fd3File);
 
-    child.on("error", (e) => {
-      tryClose(fd3File);
-      rej(e);
-    });
-    child.on("close", (code) => {
-      stderrLog.end();
-      tryClose(fd3File);
-      const raw = readAndUnlink(fd3Path);
-      const details = [];
-      parseFd3Buffer(raw, details);
-      const exitCode = typeof code === "number" ? code : -1;
-      res({
-        verdict: exitCode === 0 ? "pass" : "fail",
-        details,
-        exitCode,
-      });
-    });
-  });
+  const raw = readAndUnlink(fsSync, fd3Path);
+  const details = [];
+  parseFd3Buffer(raw, details);
+  const exitCode = typeof code === "number" ? code : -1;
+  return {
+    verdict: exitCode === 0 ? "pass" : "fail",
+    details,
+    exitCode,
+  };
 }
 
 function pushRow(line, details) {
@@ -103,23 +91,23 @@ function pushRow(line, details) {
   }
 }
 
-function tryClose(fd) {
+function tryClose(fsSync, fd) {
   try {
-    closeSync(fd);
+    fsSync.closeSync(fd);
   } catch {
     // already closed
   }
 }
 
-function readAndUnlink(path) {
+function readAndUnlink(fsSync, path) {
   let raw = "";
   try {
-    raw = readFileSync(path, "utf8");
+    raw = fsSync.readFileSync(path, "utf8");
   } catch {
     // empty
   }
   try {
-    unlinkSync(path);
+    fsSync.unlinkSync(path);
   } catch {
     // best-effort cleanup
   }

package/src/benchmark/judge.js

@@ -21,9 +21,6 @@
  * historical run from its judge.ndjson file).
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { readFile } from "node:fs/promises";
-import { createInterface } from "node:readline";
 import { createJudge } from "../judge.js";
 import { createRedactor } from "../redaction.js";
 
@@ -45,12 +42,15 @@ import { createRedactor } from "../redaction.js";
  * @param {import("./task-family.js").Task} task
  * @param {import("./workdir.js").Workdir} workdir
  * @param {import("./invariants.js").InvariantsResult} invariants
- * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string}} deps
+ * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string, runtime: import("@forwardimpact/libutil/runtime").Runtime}} deps
  * @param {JudgeContext} [context]
  * @returns {Promise<JudgeVerdict>}
  */
 export async function runJudge(task, workdir, invariants, deps, context) {
-  const template = await readFile(task.paths.judge, "utf8");
+  const runtime = deps.runtime;
+  if (!runtime) throw new Error("runtime is required");
+  const fs = runtime.fs;
+  const template = await fs.readFile(task.paths.judge, "utf8");
   const invariantsJson = JSON.stringify(invariants, null, 2);
   const taskText = template
     .replaceAll("{{INVARIANTS_RESULT}}", invariantsJson)
@@ -61,7 +61,7 @@ export async function runJudge(task, workdir, invariants, deps, context) {
     .replaceAll("{{TASK_ID}}", task.id)
     .replaceAll("{{TASK_DIR}}", workdir.cwd);
 
-  const output = createWriteStream(workdir.judgeTracePath);
+  const output = fs.createWriteStream(workdir.judgeTracePath);
   const judge = createJudge({
     cwd: workdir.cwd,
     query: deps.query,
@@ -70,7 +70,8 @@ export async function runJudge(task, workdir, invariants, deps, context) {
     judgeProfile: deps.judgeProfile,
     profilesDir: deps.profilesDir,
     maxTurns: 25,
-    redactor: createRedactor(),
+    redactor: createRedactor({ runtime }),
+    runtime,
   });
 
   let outcome;
@@ -95,13 +96,14 @@ export async function runJudge(task, workdir, invariants, deps, context) {
  * and map the verdict (`success → pass`, `failure → fail`). Preserved for
  * offline analysis; not used on the runtime happy path.
  * @param {string} tracePath
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<JudgeVerdict | null>}
  */
-export async function parseConcludeFromTrace(tracePath) {
-  const stream = createReadStream(tracePath);
-  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+export async function parseConcludeFromTrace(tracePath, runtime) {
+  if (!runtime) throw new Error("runtime is required");
+  const content = await runtime.fs.readFile(tracePath, "utf8");
   let last = null;
-  for await (const line of rl) {
+  for (const line of content.split("\n")) {
     const candidate = extractConcludeInput(line);
     if (candidate) last = candidate;
   }