@forwardimpact/libeval 0.1.51 → 0.1.52

package/src/benchmark/runner.js

@@ -14,8 +14,6 @@
  * the JSONL append is the system of record.
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, readFile, unlink } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { join, resolve as resolvePath } from "node:path";
 
@@ -60,17 +58,21 @@ export class BenchmarkRunner {
    *   write a valid NDJSON trace to `workdir.agentTracePath`. Default uses
    *   `createAgentRunner` with the harness `BASE_TOOLS` allowlist. Internal
    *   testing only — not part of the public API.
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} opts.runtime -
+   *   Injected ambient collaborators (`fs`, `subprocess`, `clock`, `proc`),
+   *   threaded into the installers, workdir manager, invariants, and judge.
    * @param {Function} [opts.runInvariants] - Test seam: replaces `runInvariants`.
-   *   Same contract as `runInvariants(task, ctx)`. Internal testing only.
+   *   Same contract as `runInvariants(task, ctx, runtime)`. Internal testing only.
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
-   *   contract as `runJudge(task, workdir, invariants, deps)`. Internal testing
-   *   only.
+   *   contract as `runJudge(task, workdir, invariants, deps)` (deps carries
+   *   `runtime`). Internal testing only.
    * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
-   *   Same contract as `installApm(family, outputDir)`. Lets tests inject a
-   *   fake `apm` spawn (or skip the install entirely) so the suite never
-   *   shells out to a real `apm` binary. Internal testing only.
+   *   Same contract as `installApm(family, outputDir, runtime)`. Lets tests
+   *   inject a fake subprocess (or skip the install entirely) so the suite
+   *   never shells out to a real `apm` binary. Internal testing only.
    * @param {Function} [opts.installNpm] - Test seam: replaces `installNpm`.
-   *   Same contract as `installNpm(family, stagingDir)`. Internal testing only.
+   *   Same contract as `installNpm(family, stagingDir, runtime)`. Internal
+   *   testing only.
    */
   constructor({
     family,
@@ -84,6 +86,7 @@ export class BenchmarkRunner {
     allowedTools,
     maxTurns,
     termGraceMs,
+    runtime,
     // Test seams — default to the real implementations.
     runAgent,
     runInvariants: runInvariantsHook,
@@ -91,12 +94,8 @@ export class BenchmarkRunner {
     installApm: installApmHook,
     installNpm: installNpmHook,
   }) {
-    if (!family) throw new Error("family is required");
-    if (!Number.isInteger(runs) || runs < 1)
-      throw new Error("runs must be an integer ≥ 1");
-    if (!output) throw new Error("output is required");
-    if (!agentModel) throw new Error("agentModel is required");
-    if (!query) throw new Error("query is required");
+    validateRunnerArgs({ family, runs, output, agentModel, query, runtime });
+    this.runtime = runtime;
     this.familyInput = family;
     this.runs = runs;
     this.output = output;
@@ -123,15 +122,16 @@ export class BenchmarkRunner {
    * @returns {AsyncGenerator<object>}
    */
   async *run() {
+    const runtime = this.runtime;
     const family =
       typeof this.familyInput === "string"
-        ? await loadTaskFamily(this.familyInput)
+        ? await loadTaskFamily(this.familyInput, runtime)
         : this.familyInput;
 
-    await mkdir(this.output, { recursive: true });
+    await runtime.fs.mkdir(this.output, { recursive: true });
     const { stagingDir, skillSetHash, judgeProfilesDir } =
-      await this._installApmHook(family, this.output);
-    await this._installNpmHook(family, stagingDir);
+      await this._installApmHook(family, this.output, runtime);
+    await this._installNpmHook(family, stagingDir, runtime);
 
     const tasks = family.tasks();
     if (this.profiles.judge) {
@@ -139,6 +139,7 @@ export class BenchmarkRunner {
         family,
         judgeProfilesDir,
         this.profiles.judge,
+        runtime,
       );
     }
 
@@ -147,10 +148,13 @@ export class BenchmarkRunner {
       runOutputDir: this.output,
       termGraceMs: this.termGraceMs,
       familyRootPath: family.rootPath,
+      runtime,
     });
 
     const resultsPath = join(this.output, "results.jsonl");
-    const resultsStream = createWriteStream(resultsPath, { flags: "a" });
+    const resultsStream = runtime.fs.createWriteStream(resultsPath, {
+      flags: "a",
+    });
     try {
       for (const task of tasks) {
         for (let runIndex = 0; runIndex < this.runs; runIndex++) {
@@ -172,7 +176,7 @@ export class BenchmarkRunner {
   }
 
   async #runOne(family, wm, task, runIndex, skillSetHash, judgeProfilesDir) {
-    const t0 = Date.now();
+    const t0 = this.runtime.clock.now();
     const workdir = await wm.start(task, runIndex);
     try {
       if (workdir.preflightError) {
@@ -182,7 +186,7 @@ export class BenchmarkRunner {
           workdir,
           skillSetHash,
           familyRevision: family.familyRevision,
-          durationMs: Date.now() - t0,
+          durationMs: this.runtime.clock.now() - t0,
         });
         return this.#validateOrFallback(
           record,
@@ -191,11 +195,15 @@ export class BenchmarkRunner {
       }
       const { costUsd, turns, submission, agentError } =
         await this.#runAgentSafe(task, workdir);
-      const invariants = await this._runInvariantsHook(task, {
-        cwd: workdir.cwd,
-        port: workdir.port,
-        runDir: workdir.runDir,
-      });
+      const invariants = await this._runInvariantsHook(
+        task,
+        {
+          cwd: workdir.cwd,
+          port: workdir.port,
+          runDir: workdir.runDir,
+        },
+        this.runtime,
+      );
       let judgeVerdict = null;
       if (task.paths.judge) {
         const judgeContext = await this.#buildJudgeContext(
@@ -212,6 +220,7 @@ export class BenchmarkRunner {
             model: this.judgeModel,
             judgeProfile: this.profiles.judge ?? undefined,
             profilesDir: judgeProfilesDir,
+            runtime: this.runtime,
           },
           judgeContext,
         );
@@ -245,7 +254,7 @@ export class BenchmarkRunner {
         },
         skillSetHash,
         familyRevision: family.familyRevision,
-        durationMs: Date.now() - t0,
+        durationMs: this.runtime.clock.now() - t0,
         ...(agentError && { agentError }),
       };
       return this.#validateOrFallback(record, resultsRecordKey(task, runIndex));
@@ -283,10 +292,11 @@ export class BenchmarkRunner {
    * agent.ndjson and supervisor.ndjson and extract cost/turns/submission.
    */
   async #runAgent(task, workdir) {
+    const fs = this.runtime.fs;
     const combinedPath = join(workdir.runDir, ".combined.ndjson");
-    const combinedStream = createWriteStream(combinedPath);
+    const combinedStream = fs.createWriteStream(combinedPath);
     const supervisorInstructions = task.paths.supervisor
-      ? await readFile(task.paths.supervisor, "utf8").catch(() => null)
+      ? await fs.readFile(task.paths.supervisor, "utf8").catch(() => null)
       : null;
     const supervisor = createSupervisor({
       supervisorCwd: workdir.cwd,
@@ -301,9 +311,11 @@ export class BenchmarkRunner {
       ...(supervisorInstructions && { taskAmend: supervisorInstructions }),
       redactor: createRedactor({
         allowlist: [...DEFAULT_ENV_ALLOWLIST, ...(workdir.envNames ?? [])],
+        runtime: this.runtime,
       }),
+      runtime: this.runtime,
     });
-    const instructions = await readFile(task.paths.instructions, "utf8");
+    const instructions = await fs.readFile(task.paths.instructions, "utf8");
     let agentError = null;
     try {
       const result = await supervisor.run(instructions);
@@ -316,16 +328,21 @@ export class BenchmarkRunner {
       await new Promise((r) => combinedStream.end(r));
     }
     const summary = await splitAndSummarize(
+      this.runtime,
       combinedPath,
       workdir.agentTracePath,
       workdir.supervisorTracePath,
     );
-    await unlink(combinedPath).catch(() => {});
+    await fs.unlink(combinedPath).catch(() => {});
     return { ...summary, agentError };
   }
 
   async #buildJudgeContext(task, workdir, skillSetHash) {
-    const agentInstructions = await readFile(task.paths.instructions, "utf8");
+    const fs = this.runtime.fs;
+    const agentInstructions = await fs.readFile(
+      task.paths.instructions,
+      "utf8",
+    );
     let agentProfile = "";
     if (this.profiles.agent) {
       const profilePath = resolvePath(
@@ -333,7 +350,7 @@ export class BenchmarkRunner {
         ".claude/agents",
         `${this.profiles.agent}.md`,
       );
-      agentProfile = await readFile(profilePath, "utf8").catch(() => "");
+      agentProfile = await fs.readFile(profilePath, "utf8").catch(() => "");
     }
     return { agentInstructions, agentProfile, skillSetHash };
   }
@@ -390,6 +407,27 @@ export class BenchmarkRunner {
   }
 }
 
+/**
+ * Validate the required BenchmarkRunner constructor arguments. Extracted from
+ * the constructor to keep its cognitive complexity under the lint ceiling.
+ */
+function validateRunnerArgs({
+  family,
+  runs,
+  output,
+  agentModel,
+  query,
+  runtime,
+}) {
+  if (!family) throw new Error("family is required");
+  if (!Number.isInteger(runs) || runs < 1)
+    throw new Error("runs must be an integer ≥ 1");
+  if (!output) throw new Error("output is required");
+  if (!agentModel) throw new Error("agentModel is required");
+  if (!query) throw new Error("query is required");
+  if (!runtime) throw new Error("runtime is required");
+}
+
 function resultsRecordKey(task, runIndex) {
   return { taskId: task.id, runIndex };
 }
@@ -408,11 +446,17 @@ async function writeRecord(stream, record) {
  * `supervisorPath`.
  */
 // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
-async function splitAndSummarize(combinedPath, agentPath, supervisorPath) {
-  const agentStream = createWriteStream(agentPath);
-  const supStream = createWriteStream(supervisorPath);
+async function splitAndSummarize(
+  runtime,
+  combinedPath,
+  agentPath,
+  supervisorPath,
+) {
+  const fs = runtime.fs;
+  const agentStream = fs.createWriteStream(agentPath);
+  const supStream = fs.createWriteStream(supervisorPath);
   const rl = createInterface({
-    input: createReadStream(combinedPath),
+    input: fs.createReadStream(combinedPath),
     crlfDelay: Infinity,
   });
   let agentCost = 0;

package/src/benchmark/task-family.js

@@ -17,45 +17,55 @@
  * a temp dir and `familyRevision` becomes `git:<sha>` of HEAD at clone time.
  * Local paths use the canonical-tree algorithm from design § Family revision
  * algorithm so the result is stable across operating systems.
+ *
+ * Filesystem and subprocess access route through the injected `runtime` bag
+ * (`runtime.fs` async, `runtime.subprocess.run` one-shot, `tmpdir` derived
+ * from `runtime.proc.env`).
  */
 
-import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import {
-  access,
-  constants,
-  lstat,
-  mkdtemp,
-  readdir,
-  readFile,
-  realpath,
-} from "node:fs/promises";
-import { tmpdir } from "node:os";
 import { join, posix, relative, resolve, sep } from "node:path";
 
 const GIT_URL_RE = /^(git@|https?:\/\/|ssh:\/\/|git:\/\/)/;
 const SKIP_DIRS = new Set([".git", "node_modules"]);
+// POSIX `X_OK` (execute permission); node's fs honours the numeric mode, so we
+// avoid importing `node:fs`'s `constants` (which would light the fs smell).
+const X_OK = 1;
+
+/**
+ * Derive the system temp dir from the env (node's `os.tmpdir()` is itself an
+ * env-respecting wrapper). The runtime bag has no `os` slot by design.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {string}
+ */
+function tmpdir(runtime) {
+  return runtime.proc.env.TMPDIR ?? "/tmp";
+}
 
 /**
  * Load a task family from a local path or git URL.
  * @param {string} rootPathOrGitUrl
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<TaskFamily>}
  */
-export async function loadTaskFamily(rootPathOrGitUrl) {
+export async function loadTaskFamily(rootPathOrGitUrl, runtime) {
+  if (!runtime) throw new Error("runtime is required");
   const isGit = GIT_URL_RE.test(rootPathOrGitUrl);
   let rootPath;
   let familyRevision;
   if (isGit) {
-    const dir = await mkdtemp(join(tmpdir(), "fit-benchmark-family-"));
-    await gitClone(rootPathOrGitUrl, dir);
+    const dir = await runtime.fs.mkdtemp(
+      join(tmpdir(runtime), "fit-benchmark-family-"),
+    );
+    await gitClone(runtime, rootPathOrGitUrl, dir);
     rootPath = dir;
-    familyRevision = "git:" + (await gitHead(dir));
+    familyRevision = "git:" + (await gitHead(runtime, dir));
   } else {
     rootPath = resolve(rootPathOrGitUrl);
-    familyRevision = "sha256:" + (await canonicalTreeHash(rootPath));
+    familyRevision = "sha256:" + (await canonicalTreeHash(runtime, rootPath));
   }
 
-  const tasks = await discoverTasks(rootPath);
+  const tasks = await discoverTasks(runtime, rootPath);
 
   return {
     rootPath,
@@ -73,27 +83,30 @@ export async function loadTaskFamily(rootPathOrGitUrl) {
  * @param {TaskFamily} _family
  * @param {string} judgeProfilesDir
  * @param {string} judgeProfile
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<void>}
  */
 export async function assertJudgeProfileStaged(
   _family,
   judgeProfilesDir,
   judgeProfile,
+  runtime,
 ) {
   const candidate = join(judgeProfilesDir, `${judgeProfile}.md`);
   try {
-    await access(candidate);
+    await runtime.fs.access(candidate);
   } catch {
     throw new Error(`judge profile not staged: ${candidate}`);
   }
 }
 
-async function discoverTasks(rootPath) {
+async function discoverTasks(runtime, rootPath) {
+  const fs = runtime.fs;
   const tasksRoot = join(rootPath, "tasks");
   const tasks = [];
   let entries;
   try {
-    entries = await readdir(tasksRoot, { withFileTypes: true });
+    entries = await fs.readdir(tasksRoot, { withFileTypes: true });
   } catch (e) {
     if (e.code === "ENOENT") return tasks;
     throw e;
@@ -110,11 +123,15 @@ async function discoverTasks(rootPath) {
       paths: {
         taskDir,
         instructions: join(taskDir, "agent.task.md"),
-        supervisor: (await fileExists(supervisorPath)) ? supervisorPath : null,
-        judge: (await fileExists(judgePath)) ? judgePath : null,
+        supervisor: (await fileExists(fs, supervisorPath))
+          ? supervisorPath
+          : null,
+        judge: (await fileExists(fs, judgePath)) ? judgePath : null,
         hooks: join(taskDir, "hooks"),
-        preflight: (await fileExecutable(preflightPath)) ? preflightPath : null,
-        invariants: (await fileExecutable(invariantsPath))
+        preflight: (await fileExecutable(fs, preflightPath))
+          ? preflightPath
+          : null,
+        invariants: (await fileExecutable(fs, invariantsPath))
           ? invariantsPath
           : null,
         specs: join(taskDir, "specs"),
@@ -126,18 +143,18 @@ async function discoverTasks(rootPath) {
   return tasks;
 }
 
-async function fileExists(path) {
+async function fileExists(fs, path) {
   try {
-    await access(path);
+    await fs.access(path);
     return true;
   } catch {
     return false;
   }
 }
 
-async function fileExecutable(path) {
+async function fileExecutable(fs, path) {
   try {
-    await access(path, constants.X_OK);
+    await fs.access(path, X_OK);
     return true;
   } catch {
     return false;
@@ -151,16 +168,18 @@ async function fileExecutable(path) {
  *   sort by NFC-normalised POSIX-style root-relative path
  *   row = <rel-path>\0<hex-sha256>\n
  *   sha256(concat(rows))
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @param {string} rootPath
  * @returns {Promise<string>} hex digest
  */
-async function canonicalTreeHash(rootPath) {
-  const real = await realpath(rootPath);
+async function canonicalTreeHash(runtime, rootPath) {
+  const fs = runtime.fs;
+  const real = await fs.realpath(rootPath);
   const rows = [];
-  for await (const filePath of walkFiles(real)) {
+  for await (const filePath of walkFiles(fs, real)) {
     const rel = toPosix(relative(real, filePath)).normalize("NFC");
-    const target = await realpath(filePath);
-    const bytes = await readFile(target);
+    const target = await fs.realpath(filePath);
+    const bytes = await fs.readFile(target);
     const hex = createHash("sha256").update(bytes).digest("hex");
     rows.push({ rel, hex });
   }
@@ -170,15 +189,15 @@ async function canonicalTreeHash(rootPath) {
   return acc.digest("hex");
 }
 
-async function* walkFiles(dir) {
-  const entries = await readdir(dir, { withFileTypes: true });
+async function* walkFiles(fs, dir) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
   for (const entry of entries) {
     const full = join(dir, entry.name);
     if (entry.isDirectory()) {
       if (SKIP_DIRS.has(entry.name)) continue;
-      yield* walkFiles(full);
+      yield* walkFiles(fs, full);
     } else if (entry.isSymbolicLink()) {
-      const resolvedFile = await resolveSymlinkToFile(full);
+      const resolvedFile = await resolveSymlinkToFile(fs, full);
       if (resolvedFile) yield full;
     } else if (entry.isFile()) {
       yield full;
@@ -190,12 +209,12 @@ async function* walkFiles(dir) {
  * Return the resolved path if `linkPath` is a symlink to a regular file.
  * Returns null for dangling symlinks or links to non-file targets.
  */
-async function resolveSymlinkToFile(linkPath) {
-  const st = await lstat(linkPath);
+async function resolveSymlinkToFile(fs, linkPath) {
+  const st = await fs.lstat(linkPath);
   if (!st.isSymbolicLink()) return null;
   try {
-    const resolved = await realpath(linkPath);
-    const tstat = await lstat(resolved);
+    const resolved = await fs.realpath(linkPath);
+    const tstat = await fs.lstat(resolved);
     return tstat.isFile() ? resolved : null;
   } catch {
     return null;
@@ -207,32 +226,24 @@ function toPosix(p) {
   return p.split(sep).join(posix.sep);
 }
 
-async function gitClone(url, dir) {
-  await run("git", ["clone", "--depth", "1", url, dir]);
+async function gitClone(runtime, url, dir) {
+  await git(runtime, ["clone", "--depth", "1", url, dir]);
 }
 
-async function gitHead(dir) {
-  const out = await run("git", ["-C", dir, "rev-parse", "HEAD"]);
+async function gitHead(runtime, dir) {
+  const out = await git(runtime, ["-C", dir, "rev-parse", "HEAD"]);
   return out.trim();
 }
 
-function run(cmd, args) {
-  return new Promise((res, rej) => {
-    const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (d) => {
-      stdout += d.toString();
-    });
-    child.stderr.on("data", (d) => {
-      stderr += d.toString();
-    });
-    child.on("error", rej);
-    child.on("close", (code) => {
-      if (code === 0) res(stdout);
-      else rej(new Error(`${cmd} ${args.join(" ")} exited ${code}: ${stderr}`));
-    });
-  });
+async function git(runtime, args) {
+  const { stdout, stderr, exitCode } = await runtime.subprocess.run(
+    "git",
+    args,
+  );
+  if (exitCode !== 0) {
+    throw new Error(`git ${args.join(" ")} exited ${exitCode}: ${stderr}`);
+  }
+  return stdout;
 }
 
 /**