@forwardimpact/libeval 0.1.38 → 0.1.39

package/bin/fit-benchmark.js

@@ -68,6 +68,11 @@ export const definition = {
           description:
             "Agent-under-test turn budget (default: 50, 0 = unlimited)",
         },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
       },
     },
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.38",
+  "version": "0.1.39",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/benchmark/apm-installer.js

@@ -19,19 +19,32 @@ export async function installApm(family, outputDir) {
   const stagingDir = join(outputDir, ".apm-staging");
   const stagedClaude = join(stagingDir, ".claude");
   const sourceClaude = join(family.rootPath, ".claude");
+  const apmYml = join(family.rootPath, "apm.yml");
 
-  await runApmInstall(family.rootPath);
+  const hasApm = await access(apmYml)
+    .then(() => true)
+    .catch(() => false);
 
-  try {
-    await access(sourceClaude);
-  } catch {
-    throw new Error(
-      `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
-    );
+  if (hasApm) {
+    await runApmInstall(family.rootPath);
+    try {
+      await access(sourceClaude);
+    } catch {
+      throw new Error(
+        `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
+      );
+    }
   }
 
   await rm(stagingDir, { recursive: true, force: true });
-  await cp(sourceClaude, stagedClaude, { recursive: true });
+  const hasClaudeDir = await access(sourceClaude)
+    .then(() => true)
+    .catch(() => false);
+  if (hasClaudeDir) {
+    await cp(sourceClaude, stagedClaude, { recursive: true });
+  } else {
+    await mkdir(stagedClaude, { recursive: true });
+  }
 
   // Stage the family-local judge profile outside .claude/ so it is available
   // to the judge but never copied into the agent-under-test's CWD.
@@ -44,12 +57,15 @@ export async function installApm(family, outputDir) {
   } catch {}
 
   const lockPath = join(family.rootPath, "apm.lock.yaml");
-  const lockBytes = await readFile(lockPath).catch(() => {
-    throw new Error(`apm install did not produce apm.lock.yaml at ${lockPath}`);
-  });
-  const skillSetHash =
-    "sha256:" +
-    createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
+  let skillSetHash = "";
+  try {
+    const lockBytes = await readFile(lockPath);
+    skillSetHash =
+      "sha256:" +
+      createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
+  } catch {
+    // No lockfile — family doesn't use skill packs.
+  }
 
   return { stagingDir, skillSetHash, judgeProfilesDir };
 }

package/src/benchmark/env-loader.js

@@ -0,0 +1,146 @@
+/**
+ * Env-loader — auto-discover `.env` / `.env.local` files in a task family
+ * and its tasks, load them into `process.env`, and render the merged result
+ * into each agent CWD.
+ *
+ * Discovery paths (loaded in this order, first value per key wins):
+ *   1. process.env  (CI secrets, shell env — never overwritten)
+ *   2. <family>/.env.local
+ *   3. <family>/.env
+ *   4. tasks/<id>/.env.local
+ *   5. tasks/<id>/.env
+ *
+ * Every discovered env file — family or task — is loaded into process.env
+ * AND rendered (with resolved values) into the agent working directory.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const ENV_FILES = [".env.local", ".env"];
+
+/**
+ * Parse a `.env` file into an array of {key, value} pairs.
+ * Handles KEY=VALUE, # comments, blank lines, and single/double-quoted values.
+ * @param {string} content
+ * @returns {Array<{key: string, value: string}>}
+ */
+export function parseEnvFile(content) {
+  const entries = [];
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line || line.startsWith("#")) continue;
+    const eq = line.indexOf("=");
+    if (eq === -1) continue;
+    const key = line.slice(0, eq).trim();
+    if (!key) continue;
+    let value = line.slice(eq + 1).trim();
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    entries.push({ key, value });
+  }
+  return entries;
+}
+
+/**
+ * Read and parse an env file, returning [] if the file does not exist.
+ * @param {string} filePath
+ * @returns {Promise<Array<{key: string, value: string}>>}
+ */
+async function readEnvFile(filePath) {
+  try {
+    const content = await readFile(filePath, "utf8");
+    return parseEnvFile(content);
+  } catch (e) {
+    if (e.code === "ENOENT") return [];
+    throw e;
+  }
+}
+
+/**
+ * Load entries into process.env. Existing keys are never overwritten.
+ * @param {Array<{key: string, value: string}>} entries
+ * @returns {string[]} var names that were loaded
+ */
+function applyToProcessEnv(entries) {
+  const names = [];
+  for (const { key, value } of entries) {
+    names.push(key);
+    if (process.env[key] === undefined) {
+      process.env[key] = value;
+    }
+  }
+  return names;
+}
+
+/**
+ * Load one env file: apply to process.env, record keys in the merged map.
+ * @param {string} dir
+ * @param {string} file
+ * @param {Set<string>} names
+ * @param {Map<string, Map<string, true>>} merged
+ */
+async function loadOneEnvFile(dir, file, names, merged) {
+  const entries = await readEnvFile(join(dir, file));
+  if (entries.length === 0) return;
+  for (const name of applyToProcessEnv(entries)) names.add(name);
+  if (!merged.has(file)) merged.set(file, new Map());
+  const fileMap = merged.get(file);
+  for (const { key } of entries) {
+    if (!fileMap.has(key)) fileMap.set(key, true);
+  }
+}
+
+/**
+ * Scan directories for env files, load into process.env, and collect
+ * a merged key manifest per filename.
+ * @param {string[]} dirs
+ * @returns {Promise<{names: Set<string>, merged: Map<string, Map<string, true>>}>}
+ */
+async function collectEnvEntries(dirs) {
+  const names = new Set();
+  const merged = new Map();
+  for (const dir of dirs) {
+    for (const file of ENV_FILES) {
+      await loadOneEnvFile(dir, file, names, merged);
+    }
+  }
+  return { names, merged };
+}
+
+/**
+ * Write resolved env files into the agent CWD and warn about empty values.
+ * @param {Map<string, Map<string, true>>} merged
+ * @param {string} agentCwd
+ */
+async function renderEnvFiles(merged, agentCwd) {
+  for (const [file, keyMap] of merged) {
+    const keys = [...keyMap.keys()];
+    const resolved = keys.map((key) => `${key}=${process.env[key] ?? ""}`);
+    await writeFile(join(agentCwd, file), resolved.join("\n") + "\n");
+    const empty = keys.filter((key) => !process.env[key]);
+    if (empty.length > 0) {
+      process.stderr.write(
+        `libeval: env warning: ${file} declares vars with no value: ${empty.join(", ")}\n`,
+      );
+    }
+  }
+}
+
+/**
+ * Discover `.env` / `.env.local` in one or more directories, load them
+ * into process.env, and render the resolved values into the agent CWD.
+ *
+ * @param {string[]} dirs - Directories to scan (family root, task dir, etc.)
+ * @param {string} agentCwd - Agent working directory to render into.
+ * @returns {Promise<string[]>} All var names discovered (for redaction).
+ */
+export async function loadEnv(dirs, agentCwd) {
+  const { names, merged } = await collectEnvEntries(dirs);
+  await renderEnvFiles(merged, agentCwd);
+  return [...names];
+}

package/src/benchmark/result.js

@@ -29,7 +29,7 @@ const JUDGE_VERDICT_SHAPE = z.object({
 
 const PROFILES_SHAPE = z.object({
   agent: z.union([z.string(), z.null()]),
-  supervisor: z.null(),
+  supervisor: z.union([z.string(), z.null()]),
   judge: z.union([z.string(), z.null()]),
 });
 
@@ -48,8 +48,8 @@ const COMMON_FIELDS = {
   profiles: PROFILES_SHAPE,
   model: z.object({
     agent: z.string(),
-    supervisor: z.string(),
-    judge: z.string(),
+    supervisor: z.string().optional(),
+    judge: z.string().optional(),
   }),
   skillSetHash: z.string(),
   familyRevision: z.string(),
@@ -65,7 +65,7 @@ const HAPPY_RECORD = z.object({
   ...COMMON_FIELDS,
   scoring: SCORING_SHAPE,
   submission: z.string(),
-  judgeVerdict: JUDGE_VERDICT_SHAPE,
+  judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
   agentTracePath: z.string(),
   supervisorTracePath: z.string(),
   judgeTracePath: z.string(),

package/src/benchmark/runner.js

@@ -15,11 +15,11 @@
  */
 
 import { createReadStream, createWriteStream } from "node:fs";
-import { access, constants, mkdir, readFile, unlink } from "node:fs/promises";
+import { mkdir, readFile, unlink } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { join, resolve as resolvePath } from "node:path";
 
-import { createRedactor } from "../redaction.js";
+import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
 import { createSupervisor } from "../supervisor.js";
 import { installApm } from "./apm-installer.js";
 import { runJudge } from "./judge.js";
@@ -28,7 +28,16 @@ import { runScoring } from "./scorer.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
 import { createWorkdirManager } from "./workdir.js";
 
-const BASE_TOOLS = ["Bash", "Read", "Glob", "Grep", "Write", "Edit"];
+const BASE_TOOLS = [
+  "Bash",
+  "Read",
+  "Glob",
+  "Grep",
+  "Write",
+  "Edit",
+  "Agent",
+  "TodoWrite",
+];
 
 /** Sole orchestrator for a task-family benchmark run. */
 export class BenchmarkRunner {
@@ -42,6 +51,7 @@ export class BenchmarkRunner {
    * @param {string} opts.judgeModel
    * @param {{agent?: string, judge?: string}} [opts.profiles]
    * @param {Function} opts.query - SDK query (injected for testability).
+   * @param {string[]} [opts.allowedTools] - Agent tool allowlist (default: BASE_TOOLS).
    * @param {number} [opts.maxTurns] - Agent-under-test turn budget.
    * @param {number} [opts.termGraceMs] - SIGTERM→SIGKILL grace (ms) for the per-task process group.
    * @param {Function} [opts.runAgent] - Test seam: replaces the agent-under-test
@@ -64,6 +74,7 @@ export class BenchmarkRunner {
     judgeModel,
     profiles,
     query,
+    allowedTools,
     maxTurns,
     termGraceMs,
     // Test seams — default to the real implementations.
@@ -76,8 +87,6 @@ export class BenchmarkRunner {
       throw new Error("runs must be an integer ≥ 1");
     if (!output) throw new Error("output is required");
     if (!agentModel) throw new Error("agentModel is required");
-    if (!supervisorModel) throw new Error("supervisorModel is required");
-    if (!judgeModel) throw new Error("judgeModel is required");
     if (!query) throw new Error("query is required");
     this.familyInput = family;
     this.runs = runs;
@@ -85,6 +94,7 @@ export class BenchmarkRunner {
     this.agentModel = agentModel;
     this.supervisorModel = supervisorModel;
     this.judgeModel = judgeModel;
+    this.allowedTools = allowedTools ?? BASE_TOOLS;
     this.profiles = {
       agent: profiles?.agent ?? null,
       judge: profiles?.judge ?? null,
@@ -114,9 +124,6 @@ export class BenchmarkRunner {
     );
 
     const tasks = family.tasks();
-    for (const task of tasks) {
-      await assertPreflightExecutable(task);
-    }
     if (this.profiles.judge) {
       await assertJudgeProfileStaged(
         family,
@@ -129,6 +136,7 @@ export class BenchmarkRunner {
       stagingDir,
       runOutputDir: this.output,
       termGraceMs: this.termGraceMs,
+      familyRootPath: family.rootPath,
     });
 
     const resultsPath = join(this.output, "results.jsonl");
@@ -178,33 +186,38 @@ export class BenchmarkRunner {
         port: workdir.port,
         runDir: workdir.runDir,
       });
-      const judgeContext = await this.#buildJudgeContext(
-        task,
-        workdir,
-        skillSetHash,
-      );
-      const judgeVerdict = await this._runJudgeHook(
-        task,
-        workdir,
-        scoring,
-        {
-          query: this.query,
-          model: this.judgeModel,
-          judgeProfile: this.profiles.judge ?? undefined,
-          profilesDir: judgeProfilesDir,
-        },
-        judgeContext,
-      );
+      let judgeVerdict = null;
+      if (task.paths.judge) {
+        const judgeContext = await this.#buildJudgeContext(
+          task,
+          workdir,
+          skillSetHash,
+        );
+        judgeVerdict = await this._runJudgeHook(
+          task,
+          workdir,
+          scoring,
+          {
+            query: this.query,
+            model: this.judgeModel,
+            judgeProfile: this.profiles.judge ?? undefined,
+            profilesDir: judgeProfilesDir,
+          },
+          judgeContext,
+        );
+      }
+      const verdict =
+        scoring.verdict === "pass" &&
+        (judgeVerdict === null || judgeVerdict.verdict === "pass")
+          ? "pass"
+          : "fail";
       const record = {
         taskId: task.id,
         runIndex,
-        verdict:
-          scoring.verdict === "pass" && judgeVerdict.verdict === "pass"
-            ? "pass"
-            : "fail",
+        verdict,
         scoring,
         submission,
-        judgeVerdict,
+        ...(judgeVerdict && { judgeVerdict }),
         costUsd,
         turns,
         agentTracePath: workdir.agentTracePath,
@@ -262,6 +275,9 @@ export class BenchmarkRunner {
   async #runAgent(task, workdir) {
     const combinedPath = join(workdir.runDir, ".combined.ndjson");
     const combinedStream = createWriteStream(combinedPath);
+    const supervisorInstructions = task.paths.supervisor
+      ? await readFile(task.paths.supervisor, "utf8").catch(() => null)
+      : null;
     const supervisor = createSupervisor({
       supervisorCwd: workdir.cwd,
       agentCwd: workdir.cwd,
@@ -270,9 +286,12 @@ export class BenchmarkRunner {
       agentModel: this.agentModel,
       supervisorModel: this.supervisorModel,
       maxTurns: this.maxTurns ?? 50,
-      allowedTools: BASE_TOOLS,
+      allowedTools: this.allowedTools,
       ...(this.profiles.agent && { agentProfile: this.profiles.agent }),
-      redactor: createRedactor(),
+      ...(supervisorInstructions && { taskAmend: supervisorInstructions }),
+      redactor: createRedactor({
+        allowlist: [...DEFAULT_ENV_ALLOWLIST, ...(workdir.envNames ?? [])],
+      }),
     });
     const instructions = await readFile(task.paths.instructions, "utf8");
     let agentError = null;
@@ -372,23 +391,6 @@ async function writeRecord(stream, record) {
   });
 }
 
-/**
- * Pre-flight install gate. Throws synchronously if any task's preflight
- * script is missing or not executable — design § Pre-flight contract:
- * "The harness fails the family at install if any task's preflight script
- * is missing or non-executable, before any agent session starts."
- */
-async function assertPreflightExecutable(task) {
-  const path = join(task.paths.hooks, "preflight.sh");
-  try {
-    await access(path, constants.X_OK);
-  } catch (e) {
-    throw new Error(
-      `task ${task.id}: preflight script not executable at ${path} (${e.code ?? e.message})`,
-    );
-  }
-}
-
 /**
  * Split the combined supervisor trace into agent and supervisor files, and
  * extract cost, turn count, and submission in a single pass. Agent-source

package/src/benchmark/scorer.js

@@ -28,8 +28,11 @@ import { join } from "node:path";
  * @returns {Promise<ScoringResult>}
  */
 export function runScoring(task, ctx) {
+  if (!task.paths.score) {
+    return Promise.resolve({ verdict: "pass", details: [], exitCode: 0 });
+  }
   return new Promise((res, rej) => {
-    const script = join(task.paths.hooks, "score.sh");
+    const script = task.paths.score;
     const stderrLog = createWriteStream(join(ctx.runDir, "scoring.stderr.log"));
 
     // Bun's child_process pipe setup for fd >= 3 is racy under load (it

package/src/benchmark/task-family.js

@@ -5,7 +5,7 @@
  *     .claude/                # pre-staged skills + agents (P1)
  *     tasks/<task_name>/
  *       agent.task.md
- *       supervisor.task.md    # preserved for v2; not read in v1
+ *       supervisor.task.md    # optional; appended to the task as supervisor context
  *       judge.task.md
  *       hooks/                # harness-only; never copied to agent CWD
  *         preflight.sh
@@ -23,6 +23,7 @@ import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 import {
   access,
+  constants,
   lstat,
   mkdtemp,
   readdir,
@@ -100,13 +101,20 @@ async function discoverTasks(rootPath) {
   for (const entry of entries) {
     if (!entry.isDirectory()) continue;
     const taskDir = join(tasksRoot, entry.name);
+    const supervisorPath = join(taskDir, "supervisor.task.md");
+    const judgePath = join(taskDir, "judge.task.md");
+    const preflightPath = join(taskDir, "hooks", "preflight.sh");
+    const scorePath = join(taskDir, "hooks", "score.sh");
     tasks.push({
       id: entry.name,
       paths: {
+        taskDir,
         instructions: join(taskDir, "agent.task.md"),
-        supervisor: join(taskDir, "supervisor.task.md"),
-        judge: join(taskDir, "judge.task.md"),
+        supervisor: (await fileExists(supervisorPath)) ? supervisorPath : null,
+        judge: (await fileExists(judgePath)) ? judgePath : null,
         hooks: join(taskDir, "hooks"),
+        preflight: (await fileExecutable(preflightPath)) ? preflightPath : null,
+        score: (await fileExecutable(scorePath)) ? scorePath : null,
         specs: join(taskDir, "specs"),
         workdir: join(taskDir, "workdir"),
       },
@@ -116,6 +124,24 @@ async function discoverTasks(rootPath) {
   return tasks;
 }
 
+async function fileExists(path) {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function fileExecutable(path) {
+  try {
+    await access(path, constants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Canonical-tree hash per design § Family revision algorithm:
  *   list regular files (excluding .git/, node_modules/)
@@ -210,7 +236,7 @@ function run(cmd, args) {
 /**
  * @typedef {object} Task
  * @property {string} id - Task name (directory name under tasks/)
- * @property {{instructions: string, supervisor: string, judge: string, hooks: string, specs: string, workdir: string}} paths
+ * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, score: string|null, specs: string, workdir: string}} paths
  */
 
 /**

package/src/benchmark/workdir.js

@@ -13,6 +13,8 @@ import { createServer } from "node:net";
 import { connect } from "node:net";
 import { join } from "node:path";
 
+import { loadEnv } from "./env-loader.js";
+
 const DEFAULT_TERM_GRACE_MS = 5_000;
 
 /**
@@ -25,6 +27,7 @@ const DEFAULT_TERM_GRACE_MS = 5_000;
  * @property {string} agentTracePath
  * @property {string} supervisorTracePath
  * @property {string} judgeTracePath
+ * @property {string[]} [envNames] - Env var names loaded from .env files.
  * @property {{phase: string, message: string, exitCode: number}} [preflightError]
  */
 
@@ -35,12 +38,13 @@ export class WorkdirManager {
    * @param {string} deps.stagingDir - Output of `installApm(...)`.
    * @param {string} deps.runOutputDir - Root run-output directory (parent of `runs/`).
    */
-  constructor({ stagingDir, runOutputDir, termGraceMs }) {
+  constructor({ stagingDir, runOutputDir, termGraceMs, familyRootPath }) {
     if (!stagingDir) throw new Error("stagingDir is required");
     if (!runOutputDir) throw new Error("runOutputDir is required");
     this.stagingDir = stagingDir;
     this.runOutputDir = runOutputDir;
     this.termGraceMs = termGraceMs ?? DEFAULT_TERM_GRACE_MS;
+    this.familyRootPath = familyRootPath ?? null;
   }
 
   /**
@@ -67,13 +71,20 @@ export class WorkdirManager {
       recursive: true,
     });
 
+    const envDirs = [
+      ...(this.familyRootPath ? [this.familyRootPath] : []),
+      ...(task.paths.taskDir ? [task.paths.taskDir] : []),
+    ];
+    const envNames = envDirs.length > 0 ? await loadEnv(envDirs, cwd) : [];
+
     const port = await allocatePort();
     const agentTracePath = join(runDir, "agent.ndjson");
     const supervisorTracePath = join(runDir, "supervisor.ndjson");
     const judgeTracePath = join(runDir, "judge.ndjson");
 
-    const preflightScript = join(task.paths.hooks, "preflight.sh");
-    const preflight = await runPreflight(preflightScript, cwd, port);
+    const preflight = task.paths.preflight
+      ? await runPreflight(task.paths.preflight, cwd, port)
+      : { pgid: 0 };
 
     return {
       cwd,
@@ -84,6 +95,7 @@ export class WorkdirManager {
       agentTracePath,
       supervisorTracePath,
       judgeTracePath,
+      envNames,
       ...(preflight.error && { preflightError: preflight.error }),
     };
   }

package/src/commands/benchmark-run.js

@@ -47,6 +47,12 @@ function parseRunOptions(values) {
       judge: values["judge-profile"] ?? null,
     },
     maxTurns: parseMaxTurns(values["max-turns"]),
+    allowedTools: values["allowed-tools"]
+      ? values["allowed-tools"]
+          .split(",")
+          .map((s) => s.trim())
+          .filter(Boolean)
+      : undefined,
   };
 }