@forwardimpact/libeval 0.1.50 → 0.1.52

package/src/benchmark/report.js

@@ -3,7 +3,7 @@
  * records by `taskId`, and compute pass@k via the OpenAI HumanEval
  * unbiased estimator: `1 - C(n-c, k) / C(n, k)`.
  *
- * When `includeRuns` is true, each task carries per-run detail (scoring
+ * When `includeRuns` is true, each task carries per-run detail (invariant
  * checks, judge commentary, cost, duration) and the text renderer produces
  * a full markdown report instead of just the pass@k table.
  *
@@ -12,9 +12,7 @@
  * whole report.
  */
 
-import { createReadStream } from "node:fs";
 import { join } from "node:path";
-import { createInterface } from "node:readline";
 
 import { validateResultRecord } from "./result.js";
 
@@ -22,7 +20,7 @@ import { validateResultRecord } from "./result.js";
  * @typedef {object} RunDetail
  * @property {number} runIndex
  * @property {"pass"|"fail"} verdict
- * @property {{verdict: string, details: unknown[], exitCode: number}} [scoring]
+ * @property {{verdict: string, details: unknown[], exitCode: number}} [invariants]
  * @property {{verdict: string, summary: string}} [judgeVerdict]
  * @property {number} costUsd
  * @property {number} turns
@@ -41,11 +39,17 @@ import { validateResultRecord } from "./result.js";
  */
 
 /**
- * @param {{inputDir: string, kValues: number[], includeRuns?: boolean}} opts
+ * @param {{inputDir: string, kValues: number[], includeRuns?: boolean, runtime: import("@forwardimpact/libutil/runtime").Runtime}} opts
  * @returns {Promise<{tasks: TaskReport[], totals: object}>}
  */
-export async function aggregate({ inputDir, kValues, includeRuns = false }) {
-  const records = await loadRecords(inputDir);
+export async function aggregate({
+  inputDir,
+  kValues,
+  includeRuns = false,
+  runtime,
+}) {
+  if (!runtime) throw new Error("runtime is required");
+  const records = await loadRecords(inputDir, runtime);
   const grouped = groupByTask(records.records);
   const tasks = [];
   let totalRuns = 0;
@@ -112,7 +116,7 @@ function buildRunDetail(r, acc) {
   return {
     runIndex: r.runIndex,
     verdict: r.verdict,
-    ...(r.scoring && { scoring: r.scoring }),
+    ...(r.invariants && { invariants: r.invariants }),
     ...(r.judgeVerdict && { judgeVerdict: r.judgeVerdict }),
     costUsd: r.costUsd ?? 0,
     turns: r.turns ?? 0,
@@ -262,7 +266,7 @@ function renderTaskDetail(task) {
 
   lines.push("", renderRunsTable(runs));
 
-  const checks = renderScoringChecks(runs, singleRun);
+  const checks = renderInvariantChecks(runs, singleRun);
   if (checks) lines.push("", checks);
 
   const commentary = renderJudgeCommentary(runs, singleRun);
@@ -278,7 +282,7 @@ function renderRunsTable(runs) {
   const header = [
     "Run",
     "Verdict",
-    "Scoring",
+    "Invariants",
     "Judge",
     "Cost",
     "Turns",
@@ -286,10 +290,10 @@ function renderRunsTable(runs) {
   ];
   const rows = [header, header.map(() => "---")];
   for (const r of runs) {
-    const scoringCell = r.preflightError
+    const invariantsCell = r.preflightError
       ? "preflight error"
-      : r.scoring
-        ? statusIcon(r.scoring.verdict === "pass")
+      : r.invariants
+        ? statusIcon(r.invariants.verdict === "pass")
         : "—";
     const judgeCell = r.preflightError
       ? "—"
@@ -299,7 +303,7 @@ function renderRunsTable(runs) {
     rows.push([
       String(r.runIndex),
       statusIcon(r.verdict === "pass"),
-      scoringCell,
+      invariantsCell,
       judgeCell,
       formatCost(r.costUsd),
       String(r.turns),
@@ -309,15 +313,15 @@ function renderRunsTable(runs) {
   return rows.map((r) => `| ${r.join(" | ")} |`).join("\n");
 }
 
-function renderScoringChecks(runs, singleRun) {
-  const rows = collectScoringRows(runs);
+function renderInvariantChecks(runs, singleRun) {
+  const rows = collectInvariantRows(runs);
   if (!rows.length) return null;
 
   const header = singleRun
     ? ["Check", "Result", "Message"]
     : ["Run", "Check", "Result", "Message"];
   const lines = [
-    "#### Scoring Checks",
+    "#### Invariant Checks",
     "",
     `| ${header.join(" | ")} |`,
     `| ${header.map(() => "---").join(" | ")} |`,
@@ -331,11 +335,11 @@ function renderScoringChecks(runs, singleRun) {
   return lines.join("\n");
 }
 
-function collectScoringRows(runs) {
+function collectInvariantRows(runs) {
   const rows = [];
   for (const r of runs) {
-    if (!r.scoring?.details?.length) continue;
-    for (const d of r.scoring.details) {
+    if (!r.invariants?.details?.length) continue;
+    for (const d of r.invariants.details) {
       rows.push({
         run: r.runIndex,
         check: escapeCell(String(d.test ?? "(unnamed)")),
@@ -429,20 +433,30 @@ function median(arr) {
 // Record loading
 // ---------------------------------------------------------------------------
 
-async function loadRecords(inputDir) {
+async function loadRecords(inputDir, runtime) {
   const path = join(inputDir, "results.jsonl");
-  const stream = createReadStream(path);
-  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  let content;
+  try {
+    content = await runtime.fs.readFile(path, "utf8");
+  } catch (e) {
+    // Re-throw with the stack collapsed to the message line so the CLI's
+    // error rendering stays free of node-internal async `readFile` frames
+    // (matching the pre-1370 stream-error shape the golden captured).
+    const err = new Error(e.message);
+    if (e.code) err.code = e.code;
+    err.stack = `Error: ${e.message}`;
+    throw err;
+  }
   const records = [];
   let skipped = 0;
-  for await (const line of rl) {
+  for (const line of content.split("\n")) {
     const trimmed = line.trim();
     if (!trimmed) continue;
     let record;
     try {
       record = JSON.parse(trimmed);
     } catch (e) {
-      process.stderr.write(
+      runtime.proc.stderr.write(
         `benchmark report: skipped malformed JSON line — ${e.message}\n`,
       );
       skipped++;
@@ -451,7 +465,7 @@ async function loadRecords(inputDir) {
     try {
       validateResultRecord(record);
     } catch (e) {
-      process.stderr.write(
+      runtime.proc.stderr.write(
         `benchmark report: skipped record failing schema — ${describeError(e)}\n`,
       );
       skipped++;

package/src/benchmark/result.js

@@ -3,10 +3,10 @@
  *
  * Two schemas live here:
  *   - RESULT_RECORD_SCHEMA — one record per (task, runIndex) from a full
- *     benchmark run. Has a happy branch (scoring + judge present) and a
- *     pre-flight-failure branch (scoring/judgeVerdict/submission absent).
- *   - SCORING_RECORD_SCHEMA — narrower output of `benchmark-score` (P7):
- *     ad-hoc grading without a full lifecycle.
+ *     benchmark run. Has a happy branch (invariants + judge present) and a
+ *     pre-flight-failure branch (invariants/judgeVerdict/submission absent).
+ *   - INVARIANTS_RECORD_SCHEMA — narrower output of `benchmark-invariants`
+ *     (P7): ad-hoc grading without a full lifecycle.
  *
  * Validation is throw-on-mismatch so the runner can wrap every JSONL append
  * in a guard and reject schema drift at write time.
@@ -16,7 +16,7 @@ import { z } from "zod";
 
 const VERDICT_ENUM = z.enum(["pass", "fail"]);
 
-const SCORING_SHAPE = z.object({
+const INVARIANTS_SHAPE = z.object({
   verdict: VERDICT_ENUM,
   details: z.array(z.unknown()),
   exitCode: z.number().int(),
@@ -63,7 +63,7 @@ const AGENT_ERROR_SHAPE = z.object({
 
 const HAPPY_RECORD = z.object({
   ...COMMON_FIELDS,
-  scoring: SCORING_SHAPE,
+  invariants: INVARIANTS_SHAPE,
   submission: z.string(),
   judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
   agentTracePath: z.string(),
@@ -83,7 +83,7 @@ const PREFLIGHT_RECORD = z.object({
   agentTracePath: z.string(),
   supervisorTracePath: z.string(),
   judgeTracePath: z.string(),
-  scoring: z.undefined().optional(),
+  invariants: z.undefined().optional(),
   submission: z.undefined().optional(),
   judgeVerdict: z.undefined().optional(),
   agentError: z.undefined().optional(),
@@ -91,9 +91,9 @@ const PREFLIGHT_RECORD = z.object({
 
 export const RESULT_RECORD_SCHEMA = z.union([HAPPY_RECORD, PREFLIGHT_RECORD]);
 
-export const SCORING_RECORD_SCHEMA = z.object({
+export const INVARIANTS_RECORD_SCHEMA = z.object({
   taskId: z.string().min(1),
-  scoring: SCORING_SHAPE,
+  invariants: INVARIANTS_SHAPE,
   exitCode: z.number().int(),
 });
 
@@ -109,6 +109,6 @@ export function validateResultRecord(record) {
  * Throw on schema mismatch.
  * @param {object} record
  */
-export function validateScoringRecord(record) {
-  SCORING_RECORD_SCHEMA.parse(record);
+export function validateInvariantsRecord(record) {
+  INVARIANTS_RECORD_SCHEMA.parse(record);
 }

package/src/benchmark/runner.js

@@ -4,7 +4,7 @@
  * Phases per (task, runIndex):
  *   1. WorkdirManager.start → seed CWD + run pre-flight probe
  *   2. Supervisor session (agent + supervisor) → produce traces + submission
- *   3. Scorer.runScoring → exit-code-driven verdict via fd-3 NDJSON
+ *   3. Invariants.runInvariants → exit-code-driven verdict via fd-3 NDJSON
  *   4. Judge.runJudge → Conclude-driven verdict mapped to pass/fail
  *   5. WorkdirManager.teardown → process-group cleanup
  *
@@ -14,8 +14,6 @@
  * the JSONL append is the system of record.
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, readFile, unlink } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { join, resolve as resolvePath } from "node:path";
 
@@ -25,7 +23,7 @@ import { installApm as defaultInstallApm } from "./apm-installer.js";
 import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
 import { validateResultRecord } from "./result.js";
-import { runScoring } from "./scorer.js";
+import { runInvariants } from "./invariants.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
 import { createWorkdirManager } from "./workdir.js";
 
@@ -60,17 +58,21 @@ export class BenchmarkRunner {
    *   write a valid NDJSON trace to `workdir.agentTracePath`. Default uses
    *   `createAgentRunner` with the harness `BASE_TOOLS` allowlist. Internal
    *   testing only — not part of the public API.
-   * @param {Function} [opts.runScoring] - Test seam: replaces `runScoring`.
-   *   Same contract as `runScoring(task, ctx)`. Internal testing only.
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} opts.runtime -
+   *   Injected ambient collaborators (`fs`, `subprocess`, `clock`, `proc`),
+   *   threaded into the installers, workdir manager, invariants, and judge.
+   * @param {Function} [opts.runInvariants] - Test seam: replaces `runInvariants`.
+   *   Same contract as `runInvariants(task, ctx, runtime)`. Internal testing only.
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
-   *   contract as `runJudge(task, workdir, scoring, deps)`. Internal testing
-   *   only.
+   *   contract as `runJudge(task, workdir, invariants, deps)` (deps carries
+   *   `runtime`). Internal testing only.
    * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
-   *   Same contract as `installApm(family, outputDir)`. Lets tests inject a
-   *   fake `apm` spawn (or skip the install entirely) so the suite never
-   *   shells out to a real `apm` binary. Internal testing only.
+   *   Same contract as `installApm(family, outputDir, runtime)`. Lets tests
+   *   inject a fake subprocess (or skip the install entirely) so the suite
+   *   never shells out to a real `apm` binary. Internal testing only.
    * @param {Function} [opts.installNpm] - Test seam: replaces `installNpm`.
-   *   Same contract as `installNpm(family, stagingDir)`. Internal testing only.
+   *   Same contract as `installNpm(family, stagingDir, runtime)`. Internal
+   *   testing only.
    */
   constructor({
     family,
@@ -84,19 +86,16 @@ export class BenchmarkRunner {
     allowedTools,
     maxTurns,
     termGraceMs,
+    runtime,
     // Test seams — default to the real implementations.
     runAgent,
-    runScoring: runScoringHook,
+    runInvariants: runInvariantsHook,
     runJudge: runJudgeHook,
     installApm: installApmHook,
     installNpm: installNpmHook,
   }) {
-    if (!family) throw new Error("family is required");
-    if (!Number.isInteger(runs) || runs < 1)
-      throw new Error("runs must be an integer ≥ 1");
-    if (!output) throw new Error("output is required");
-    if (!agentModel) throw new Error("agentModel is required");
-    if (!query) throw new Error("query is required");
+    validateRunnerArgs({ family, runs, output, agentModel, query, runtime });
+    this.runtime = runtime;
     this.familyInput = family;
     this.runs = runs;
     this.output = output;
@@ -112,7 +111,7 @@ export class BenchmarkRunner {
     this.maxTurns = maxTurns;
     this.termGraceMs = termGraceMs;
     this._runAgentHook = runAgent ?? null;
-    this._runScoringHook = runScoringHook ?? runScoring;
+    this._runInvariantsHook = runInvariantsHook ?? runInvariants;
     this._runJudgeHook = runJudgeHook ?? runJudge;
     this._installApmHook = installApmHook ?? defaultInstallApm;
     this._installNpmHook = installNpmHook ?? defaultInstallNpm;
@@ -123,15 +122,16 @@ export class BenchmarkRunner {
    * @returns {AsyncGenerator<object>}
    */
   async *run() {
+    const runtime = this.runtime;
     const family =
       typeof this.familyInput === "string"
-        ? await loadTaskFamily(this.familyInput)
+        ? await loadTaskFamily(this.familyInput, runtime)
         : this.familyInput;
 
-    await mkdir(this.output, { recursive: true });
+    await runtime.fs.mkdir(this.output, { recursive: true });
     const { stagingDir, skillSetHash, judgeProfilesDir } =
-      await this._installApmHook(family, this.output);
-    await this._installNpmHook(family, stagingDir);
+      await this._installApmHook(family, this.output, runtime);
+    await this._installNpmHook(family, stagingDir, runtime);
 
     const tasks = family.tasks();
     if (this.profiles.judge) {
@@ -139,6 +139,7 @@ export class BenchmarkRunner {
         family,
         judgeProfilesDir,
         this.profiles.judge,
+        runtime,
       );
     }
 
@@ -147,10 +148,13 @@ export class BenchmarkRunner {
       runOutputDir: this.output,
       termGraceMs: this.termGraceMs,
       familyRootPath: family.rootPath,
+      runtime,
     });
 
     const resultsPath = join(this.output, "results.jsonl");
-    const resultsStream = createWriteStream(resultsPath, { flags: "a" });
+    const resultsStream = runtime.fs.createWriteStream(resultsPath, {
+      flags: "a",
+    });
     try {
       for (const task of tasks) {
         for (let runIndex = 0; runIndex < this.runs; runIndex++) {
@@ -172,7 +176,7 @@ export class BenchmarkRunner {
   }
 
   async #runOne(family, wm, task, runIndex, skillSetHash, judgeProfilesDir) {
-    const t0 = Date.now();
+    const t0 = this.runtime.clock.now();
     const workdir = await wm.start(task, runIndex);
     try {
       if (workdir.preflightError) {
@@ -182,7 +186,7 @@ export class BenchmarkRunner {
           workdir,
           skillSetHash,
           familyRevision: family.familyRevision,
-          durationMs: Date.now() - t0,
+          durationMs: this.runtime.clock.now() - t0,
         });
         return this.#validateOrFallback(
           record,
@@ -191,11 +195,15 @@ export class BenchmarkRunner {
       }
       const { costUsd, turns, submission, agentError } =
         await this.#runAgentSafe(task, workdir);
-      const scoring = await this._runScoringHook(task, {
-        cwd: workdir.cwd,
-        port: workdir.port,
-        runDir: workdir.runDir,
-      });
+      const invariants = await this._runInvariantsHook(
+        task,
+        {
+          cwd: workdir.cwd,
+          port: workdir.port,
+          runDir: workdir.runDir,
+        },
+        this.runtime,
+      );
       let judgeVerdict = null;
       if (task.paths.judge) {
         const judgeContext = await this.#buildJudgeContext(
@@ -206,18 +214,19 @@ export class BenchmarkRunner {
         judgeVerdict = await this._runJudgeHook(
           task,
           workdir,
-          scoring,
+          invariants,
           {
             query: this.query,
             model: this.judgeModel,
             judgeProfile: this.profiles.judge ?? undefined,
             profilesDir: judgeProfilesDir,
+            runtime: this.runtime,
           },
           judgeContext,
         );
       }
       const verdict =
-        scoring.verdict === "pass" &&
+        invariants.verdict === "pass" &&
         (judgeVerdict === null || judgeVerdict.verdict === "pass")
           ? "pass"
           : "fail";
@@ -225,7 +234,7 @@ export class BenchmarkRunner {
         taskId: task.id,
         runIndex,
         verdict,
-        scoring,
+        invariants,
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd,
@@ -245,7 +254,7 @@ export class BenchmarkRunner {
         },
         skillSetHash,
         familyRevision: family.familyRevision,
-        durationMs: Date.now() - t0,
+        durationMs: this.runtime.clock.now() - t0,
         ...(agentError && { agentError }),
       };
       return this.#validateOrFallback(record, resultsRecordKey(task, runIndex));
@@ -283,10 +292,11 @@ export class BenchmarkRunner {
    * agent.ndjson and supervisor.ndjson and extract cost/turns/submission.
    */
   async #runAgent(task, workdir) {
+    const fs = this.runtime.fs;
     const combinedPath = join(workdir.runDir, ".combined.ndjson");
-    const combinedStream = createWriteStream(combinedPath);
+    const combinedStream = fs.createWriteStream(combinedPath);
     const supervisorInstructions = task.paths.supervisor
-      ? await readFile(task.paths.supervisor, "utf8").catch(() => null)
+      ? await fs.readFile(task.paths.supervisor, "utf8").catch(() => null)
       : null;
     const supervisor = createSupervisor({
       supervisorCwd: workdir.cwd,
@@ -301,9 +311,11 @@ export class BenchmarkRunner {
       ...(supervisorInstructions && { taskAmend: supervisorInstructions }),
       redactor: createRedactor({
         allowlist: [...DEFAULT_ENV_ALLOWLIST, ...(workdir.envNames ?? [])],
+        runtime: this.runtime,
       }),
+      runtime: this.runtime,
     });
-    const instructions = await readFile(task.paths.instructions, "utf8");
+    const instructions = await fs.readFile(task.paths.instructions, "utf8");
     let agentError = null;
     try {
       const result = await supervisor.run(instructions);
@@ -316,16 +328,21 @@ export class BenchmarkRunner {
       await new Promise((r) => combinedStream.end(r));
     }
     const summary = await splitAndSummarize(
+      this.runtime,
       combinedPath,
       workdir.agentTracePath,
       workdir.supervisorTracePath,
     );
-    await unlink(combinedPath).catch(() => {});
+    await fs.unlink(combinedPath).catch(() => {});
     return { ...summary, agentError };
   }
 
   async #buildJudgeContext(task, workdir, skillSetHash) {
-    const agentInstructions = await readFile(task.paths.instructions, "utf8");
+    const fs = this.runtime.fs;
+    const agentInstructions = await fs.readFile(
+      task.paths.instructions,
+      "utf8",
+    );
     let agentProfile = "";
     if (this.profiles.agent) {
       const profilePath = resolvePath(
@@ -333,7 +350,7 @@ export class BenchmarkRunner {
         ".claude/agents",
         `${this.profiles.agent}.md`,
       );
-      agentProfile = await readFile(profilePath, "utf8").catch(() => "");
+      agentProfile = await fs.readFile(profilePath, "utf8").catch(() => "");
     }
     return { agentInstructions, agentProfile, skillSetHash };
   }
@@ -390,6 +407,27 @@ export class BenchmarkRunner {
   }
 }
 
+/**
+ * Validate the required BenchmarkRunner constructor arguments. Extracted from
+ * the constructor to keep its cognitive complexity under the lint ceiling.
+ */
+function validateRunnerArgs({
+  family,
+  runs,
+  output,
+  agentModel,
+  query,
+  runtime,
+}) {
+  if (!family) throw new Error("family is required");
+  if (!Number.isInteger(runs) || runs < 1)
+    throw new Error("runs must be an integer ≥ 1");
+  if (!output) throw new Error("output is required");
+  if (!agentModel) throw new Error("agentModel is required");
+  if (!query) throw new Error("query is required");
+  if (!runtime) throw new Error("runtime is required");
+}
+
 function resultsRecordKey(task, runIndex) {
   return { taskId: task.id, runIndex };
 }
@@ -408,11 +446,17 @@ async function writeRecord(stream, record) {
  * `supervisorPath`.
  */
 // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
-async function splitAndSummarize(combinedPath, agentPath, supervisorPath) {
-  const agentStream = createWriteStream(agentPath);
-  const supStream = createWriteStream(supervisorPath);
+async function splitAndSummarize(
+  runtime,
+  combinedPath,
+  agentPath,
+  supervisorPath,
+) {
+  const fs = runtime.fs;
+  const agentStream = fs.createWriteStream(agentPath);
+  const supStream = fs.createWriteStream(supervisorPath);
   const rl = createInterface({
-    input: createReadStream(combinedPath),
+    input: fs.createReadStream(combinedPath),
     crlfDelay: Infinity,
   });
   let agentCost = 0;