@forwardimpact/libeval 0.1.31 → 0.1.33

package/src/benchmark/task-family.js

@@ -0,0 +1,259 @@
+/**
+ * Task-family loader. A task family is a directory under
+ *   <root>/
+ *     apm.lock.yaml
+ *     .claude/                # pre-staged skills + agents (P1)
+ *     tasks/<task_family_name>/<task_name>/
+ *       instructions.md
+ *       supervisor.task.md    # preserved for v2; not read in v1
+ *       judge.task.md
+ *       specs/                # copied into agent CWD
+ *       workdir/              # copied into agent CWD (excludes scripts/)
+ *         scripts/preflight.sh
+ *       scoring/              # template-only; never copied
+ *
+ * Local paths or git URLs are both accepted; git URLs are shallow-cloned into
+ * a temp dir and `familyRevision` becomes `git:<sha>` of HEAD at clone time.
+ * Local paths use the canonical-tree algorithm from design § Family revision
+ * algorithm so the result is stable across operating systems.
+ */
+
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+  access,
+  lstat,
+  mkdtemp,
+  readdir,
+  readFile,
+  realpath,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, posix, relative, resolve, sep } from "node:path";
+
+const GIT_URL_RE = /^(git@|https?:\/\/|ssh:\/\/|git:\/\/)/;
+const SKIP_DIRS = new Set([".git", "node_modules"]);
+
+/**
+ * Load a task family from a local path or git URL.
+ * @param {string} rootPathOrGitUrl
+ * @returns {Promise<TaskFamily>}
+ */
+export async function loadTaskFamily(rootPathOrGitUrl) {
+  const isGit = GIT_URL_RE.test(rootPathOrGitUrl);
+  let rootPath;
+  let familyRevision;
+  if (isGit) {
+    const dir = await mkdtemp(join(tmpdir(), "fit-benchmark-family-"));
+    await gitClone(rootPathOrGitUrl, dir);
+    rootPath = dir;
+    familyRevision = "git:" + (await gitHead(dir));
+  } else {
+    rootPath = resolve(rootPathOrGitUrl);
+    familyRevision = "sha256:" + (await canonicalTreeHash(rootPath));
+  }
+
+  const apmLockBytes = await readApmLockBytes(rootPath);
+  const tasks = await discoverTasks(rootPath);
+
+  return {
+    rootPath,
+    familyRevision,
+    apmLockBytes,
+    tasks() {
+      return tasks;
+    },
+  };
+}
+
+/**
+ * Assert that `<stagingDir>/.claude/agents/<judgeProfile>.md` exists. Called
+ * from `BenchmarkRunner.run()` so a missing judge profile fails the family
+ * install before any agent session starts.
+ * @param {TaskFamily} _family
+ * @param {string} stagingDir
+ * @param {string} judgeProfile
+ * @returns {Promise<void>}
+ */
+export async function assertJudgeProfileStaged(
+  _family,
+  stagingDir,
+  judgeProfile,
+) {
+  const candidate = join(stagingDir, ".claude", "agents", `${judgeProfile}.md`);
+  try {
+    await access(candidate);
+  } catch {
+    throw new Error(
+      `judge profile not staged: ${candidate} (createSupervisor resolves profiles relative to <supervisorCwd>/.claude/agents)`,
+    );
+  }
+}
+
+async function readApmLockBytes(rootPath) {
+  const lockPath = join(rootPath, "apm.lock.yaml");
+  try {
+    const raw = await readFile(lockPath);
+    return normalizeLf(raw);
+  } catch (e) {
+    if (e.code === "ENOENT") {
+      throw new Error(
+        `task family missing apm.lock.yaml at ${lockPath} (matches libpack stager.js:126; .yml is not accepted)`,
+      );
+    }
+    throw e;
+  }
+}
+
+/**
+ * Replace CRLF with LF so cross-OS authored lockfiles hash identically.
+ * @param {Buffer} buf
+ * @returns {Buffer}
+ */
+function normalizeLf(buf) {
+  const out = [];
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue;
+    out.push(buf[i]);
+  }
+  return Buffer.from(out);
+}
+
+async function discoverTasks(rootPath) {
+  const tasksRoot = join(rootPath, "tasks");
+  const tasks = [];
+  let families;
+  try {
+    families = await readdir(tasksRoot, { withFileTypes: true });
+  } catch (e) {
+    if (e.code === "ENOENT") return tasks;
+    throw e;
+  }
+  for (const family of families) {
+    if (!family.isDirectory()) continue;
+    const familyDir = join(tasksRoot, family.name);
+    const entries = await readdir(familyDir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const taskDir = join(familyDir, entry.name);
+      tasks.push({
+        id: `${family.name}/${entry.name}`,
+        paths: {
+          instructions: join(taskDir, "instructions.md"),
+          supervisor: join(taskDir, "supervisor.task.md"),
+          judge: join(taskDir, "judge.task.md"),
+          specs: join(taskDir, "specs"),
+          workdir: join(taskDir, "workdir"),
+          scoring: join(taskDir, "scoring"),
+        },
+      });
+    }
+  }
+  tasks.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return tasks;
+}
+
+/**
+ * Canonical-tree hash per design § Family revision algorithm:
+ *   list regular files (excluding .git/, node_modules/)
+ *   resolve symlinks before reading
+ *   sort by NFC-normalised POSIX-style root-relative path
+ *   row = <rel-path>\0<hex-sha256>\n
+ *   sha256(concat(rows))
+ * @param {string} rootPath
+ * @returns {Promise<string>} hex digest
+ */
+async function canonicalTreeHash(rootPath) {
+  const real = await realpath(rootPath);
+  const rows = [];
+  for await (const filePath of walkFiles(real)) {
+    const rel = toPosix(relative(real, filePath)).normalize("NFC");
+    const target = await realpath(filePath);
+    const bytes = await readFile(target);
+    const hex = createHash("sha256").update(bytes).digest("hex");
+    rows.push({ rel, hex });
+  }
+  rows.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
+  const acc = createHash("sha256");
+  for (const r of rows) acc.update(`${r.rel}\0${r.hex}\n`, "utf8");
+  return acc.digest("hex");
+}
+
+async function* walkFiles(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      yield* walkFiles(full);
+    } else if (entry.isSymbolicLink()) {
+      const resolvedFile = await resolveSymlinkToFile(full);
+      if (resolvedFile) yield full;
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+/**
+ * Return the resolved path if `linkPath` is a symlink to a regular file.
+ * Returns null for dangling symlinks or links to non-file targets.
+ */
+async function resolveSymlinkToFile(linkPath) {
+  const st = await lstat(linkPath);
+  if (!st.isSymbolicLink()) return null;
+  try {
+    const resolved = await realpath(linkPath);
+    const tstat = await lstat(resolved);
+    return tstat.isFile() ? resolved : null;
+  } catch {
+    return null;
+  }
+}
+
+function toPosix(p) {
+  if (sep === posix.sep) return p;
+  return p.split(sep).join(posix.sep);
+}
+
+async function gitClone(url, dir) {
+  await run("git", ["clone", "--depth", "1", url, dir]);
+}
+
+async function gitHead(dir) {
+  const out = await run("git", ["-C", dir, "rev-parse", "HEAD"]);
+  return out.trim();
+}
+
+function run(cmd, args) {
+  return new Promise((res, rej) => {
+    const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (d) => {
+      stdout += d.toString();
+    });
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", rej);
+    child.on("close", (code) => {
+      if (code === 0) res(stdout);
+      else rej(new Error(`${cmd} ${args.join(" ")} exited ${code}: ${stderr}`));
+    });
+  });
+}
+
+/**
+ * @typedef {object} Task
+ * @property {string} id - METR-style "task_family_name/task_name"
+ * @property {{instructions: string, supervisor: string, judge: string, specs: string, workdir: string, scoring: string}} paths
+ */
+
+/**
+ * @typedef {object} TaskFamily
+ * @property {string} rootPath
+ * @property {string} familyRevision - `git:<sha>` or `sha256:<hex>`
+ * @property {Buffer} apmLockBytes - LF-normalised
+ * @property {() => Task[]} tasks
+ */

package/src/benchmark/workdir.js

@@ -0,0 +1,248 @@
+/**
+ * WorkdirManager — per-task lifecycle: create the agent CWD, seed it from the
+ * task's workdir + specs + staged .claude/, allocate a free TCP port, run
+ * the pre-flight smoke probe, and tear down the process group at end of run.
+ *
+ * The Workdir handle threads `cwd`, `port`, `pgid`, and trace paths through
+ * runAgent → score → judge → teardown.
+ */
+
+import { spawn } from "node:child_process";
+import { cp, mkdir } from "node:fs/promises";
+import { createServer } from "node:net";
+import { connect } from "node:net";
+import { join, sep } from "node:path";
+
+const PREFLIGHT_REL = join("workdir", "scripts");
+const DEFAULT_TERM_GRACE_MS = 5_000;
+
+/**
+ * @typedef {object} Workdir
+ * @property {string} cwd - Agent CWD (per-task copy).
+ * @property {string} runDir - Parent of `cwd`; holds trace/log siblings.
+ * @property {number} port - Allocated TCP port for the agent.
+ * @property {number} pgid - Process-group id captured from the preflight child.
+ * @property {*} scaffold - Reserved per design § Components; v1 sets null.
+ * @property {string} agentTracePath
+ * @property {string} judgeTracePath
+ * @property {{phase: string, message: string, exitCode: number}} [preflightError]
+ */
+
+/** Per-task workdir lifecycle: seed → preflight → teardown. */
+export class WorkdirManager {
+  /**
+   * @param {object} deps
+   * @param {string} deps.stagingDir - Output of `installApm(...)`.
+   * @param {string} deps.runOutputDir - Root run-output directory (parent of `runs/`).
+   */
+  constructor({ stagingDir, runOutputDir, termGraceMs }) {
+    if (!stagingDir) throw new Error("stagingDir is required");
+    if (!runOutputDir) throw new Error("runOutputDir is required");
+    this.stagingDir = stagingDir;
+    this.runOutputDir = runOutputDir;
+    this.termGraceMs = termGraceMs ?? DEFAULT_TERM_GRACE_MS;
+  }
+
+  /**
+   * Create the per-task working directory and run the pre-flight probe.
+   * @param {import("./task-family.js").Task} task
+   * @param {number} runIndex
+   * @returns {Promise<Workdir>}
+   */
+  async start(task, runIndex) {
+    const slug = task.id.replace("/", "__");
+    const runDir = join(this.runOutputDir, "runs", slug, String(runIndex));
+    const cwd = join(runDir, "cwd");
+    await mkdir(cwd, { recursive: true });
+
+    await cp(task.paths.workdir, cwd, {
+      recursive: true,
+      filter: (src) => !src.endsWith(sep + PREFLIGHT_REL),
+    });
+    await cp(task.paths.specs, join(cwd, "specs"), {
+      recursive: true,
+    }).catch((e) => {
+      if (e.code !== "ENOENT") throw e;
+    });
+    await cp(join(this.stagingDir, ".claude"), join(cwd, ".claude"), {
+      recursive: true,
+    });
+
+    const port = await allocatePort();
+    const agentTracePath = join(runDir, "agent.ndjson");
+    const judgeTracePath = join(runDir, "judge.ndjson");
+
+    const preflightScript = join(task.paths.workdir, "scripts", "preflight.sh");
+    const preflight = await runPreflight(preflightScript, cwd, port);
+
+    return {
+      cwd,
+      runDir,
+      port,
+      pgid: preflight.pgid,
+      scaffold: null,
+      agentTracePath,
+      judgeTracePath,
+      ...(preflight.error && { preflightError: preflight.error }),
+    };
+  }
+
+  /**
+   * Tear down the per-task process group: SIGTERM, wait, SIGKILL, then probe.
+   * @param {Workdir} workdir
+   * @returns {Promise<{portFree: boolean, descendants: number}>}
+   */
+  async teardown(workdir) {
+    if (workdir.pgid && workdir.pgid > 0) {
+      try {
+        process.kill(-workdir.pgid, "SIGTERM");
+      } catch {
+        // Process group already gone — fine.
+      }
+      await sleep(this.termGraceMs);
+      try {
+        process.kill(-workdir.pgid, "SIGKILL");
+      } catch {
+        // Already exited.
+      }
+      // Poll briefly until the process group is empty — SIGKILL returns
+      // before the kernel finishes reaping descendants.
+      await waitFor(
+        async () => (await countDescendants(workdir.pgid)) === 0,
+        2_000,
+      );
+    }
+    const portFree = await isPortFree(workdir.port);
+    const descendants = await countDescendants(workdir.pgid);
+    return { portFree, descendants };
+  }
+}
+
+/**
+ * Spawn preflight. Stays detached so we can SIGTERM the whole process group.
+ * @param {string} script
+ * @param {string} cwd - Agent CWD passed via $WORKDIR.
+ * @param {number} port - Free TCP port passed via $PORT.
+ * @returns {Promise<{pgid: number, error?: {phase: string, message: string, exitCode: number}}>}
+ */
+function runPreflight(script, cwd, port) {
+  return new Promise((res, rej) => {
+    let stderr = "";
+    const child = spawn(script, [], {
+      cwd,
+      env: { ...process.env, WORKDIR: cwd, PORT: String(port) },
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    if (child.pid === undefined) {
+      rej(new Error(`failed to spawn preflight: ${script}`));
+      return;
+    }
+    const pgid = child.pid;
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", (e) => {
+      res({
+        pgid,
+        error: {
+          phase: "preflight",
+          message: `preflight failed to spawn: ${e.message}`,
+          exitCode: -1,
+        },
+      });
+    });
+    child.on("exit", (code, signal) => {
+      if (code === 0) {
+        res({ pgid });
+        return;
+      }
+      const message = stderr.trim() || `preflight exited with signal ${signal}`;
+      res({
+        pgid,
+        error: {
+          phase: "preflight",
+          message,
+          exitCode: typeof code === "number" ? code : -1,
+        },
+      });
+    });
+  });
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}
+
+function isPortFree(port) {
+  if (!port) return Promise.resolve(true);
+  return new Promise((res) => {
+    const socket = connect({ port, host: "127.0.0.1" }, () => {
+      socket.destroy();
+      res(false);
+    });
+    socket.on("error", () => res(true));
+    socket.setTimeout(500, () => {
+      socket.destroy();
+      res(true);
+    });
+  });
+}
+
+function countDescendants(pgid) {
+  if (!pgid || pgid <= 0) return Promise.resolve(0);
+  return new Promise((res) => {
+    const child = spawn("ps", ["-o", "pid=", "-g", String(pgid)], {
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    let out = "";
+    child.stdout.on("data", (d) => {
+      out += d.toString();
+    });
+    child.on("error", () => res(0));
+    child.on("close", () => {
+      const pids = out
+        .split("\n")
+        .map((s) => s.trim())
+        .filter(Boolean)
+        .filter((s) => Number(s) !== process.pid);
+      res(pids.length);
+    });
+  });
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function waitFor(predicate, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await predicate()) return true;
+    await sleep(50);
+  }
+  return false;
+}
+
+/**
+ * Factory function — wires real dependencies.
+ * @param {ConstructorParameters<typeof WorkdirManager>[0]} deps
+ * @returns {WorkdirManager}
+ */
+export function createWorkdirManager(deps) {
+  return new WorkdirManager(deps);
+}

package/src/commands/benchmark-report.js

@@ -0,0 +1,39 @@
+/**
+ * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
+ * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
+ * to render a markdown table.
+ */
+
+import { resolve } from "node:path";
+
+import { aggregate, renderTextReport } from "../benchmark/report.js";
+
+/**
+ * @param {object} values
+ * @param {string[]} _args
+ */
+export async function runBenchmarkReportCommand(values, _args) {
+  const inputDir = values.input;
+  if (!inputDir) throw new Error("--input is required");
+  const kRaw = values.k ?? "1,3,5";
+  const kValues = kRaw.split(",").map((t) => {
+    const n = Number.parseInt(t.trim(), 10);
+    if (!Number.isFinite(n) || n < 1) {
+      throw new Error(
+        "--k must be a comma-separated list of positive integers",
+      );
+    }
+    return n;
+  });
+  const format = values.format ?? "json";
+  if (format !== "json" && format !== "text") {
+    throw new Error("--format must be 'json' or 'text'");
+  }
+
+  const report = await aggregate({ inputDir: resolve(inputDir), kValues });
+  if (format === "text") {
+    process.stdout.write(renderTextReport(report, kValues) + "\n");
+  } else {
+    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
+  }
+}

package/src/commands/benchmark-run.js

@@ -0,0 +1,53 @@
+/**
+ * `fit-benchmark run` — run every task in a family for N runs, stream each
+ * ResultRecord to stdout (one JSON line per record), and append to the
+ * canonical `<output>/results.jsonl` for the report subcommand.
+ */
+
+import { resolve } from "node:path";
+
+import { createBenchmarkRunner } from "../benchmark/runner.js";
+
+/**
+ * @param {object} values
+ * @param {string[]} _args
+ */
+export async function runBenchmarkRunCommand(values, _args) {
+  const opts = parseRunOptions(values);
+  const { query } = await import("@anthropic-ai/claude-agent-sdk");
+  const runner = createBenchmarkRunner({ ...opts, query });
+
+  let anyFail = false;
+  for await (const record of runner.run()) {
+    process.stdout.write(JSON.stringify(record) + "\n");
+    if (record.verdict !== "pass") anyFail = true;
+  }
+  process.exit(anyFail ? 1 : 0);
+}
+
+function parseRunOptions(values) {
+  const family = values.family;
+  if (!family) throw new Error("--family is required");
+  const output = values.output;
+  if (!output) throw new Error("--output is required");
+  const runs = Number.parseInt(values.runs ?? "1", 10);
+  if (!Number.isFinite(runs) || runs < 1)
+    throw new Error("--runs must be a positive integer");
+  return {
+    family,
+    runs,
+    output: resolve(output),
+    model: values.model ?? "claude-opus-4-7[1m]",
+    profiles: {
+      agent: values["agent-profile"] ?? null,
+      judge: values["judge-profile"] ?? null,
+    },
+    maxTurns: parseMaxTurns(values["max-turns"]),
+  };
+}
+
+function parseMaxTurns(raw) {
+  if (raw === undefined) return undefined;
+  if (raw === "0") return 0;
+  return Number.parseInt(raw, 10);
+}

package/src/commands/benchmark-score.js

@@ -0,0 +1,68 @@
+/**
+ * `fit-benchmark score` — score a single task against a post-run workdir
+ * directory without invoking an agent (P6/P7). Useful for re-scoring an
+ * agent's output against revised grading material.
+ */
+
+import { writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+
+import { validateScoringRecord } from "../benchmark/result.js";
+import { runScoring } from "../benchmark/scorer.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+
+/**
+ * @param {object} values
+ * @param {string[]} _args
+ */
+export async function runBenchmarkScoreCommand(values, _args) {
+  const familyInput = values.family;
+  if (!familyInput) throw new Error("--family is required");
+  const taskId = values.task;
+  if (!taskId) throw new Error("--task is required");
+  const workdirArg = values.workdir;
+  if (!workdirArg) throw new Error("--workdir is required");
+
+  const family = await loadTaskFamily(familyInput);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task) throw new Error(`task not found in family: ${taskId}`);
+
+  const runDir = resolve(workdirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await allocatePort();
+
+  const scoring = await runScoring(task, { cwd, port, runDir });
+  const record = {
+    taskId: task.id,
+    scoring,
+    exitCode: scoring.exitCode,
+  };
+  validateScoringRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    writeFileSync(resolve(values.output), line);
+  } else {
+    process.stdout.write(line);
+  }
+  process.exit(scoring.verdict === "pass" ? 0 : 1);
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}

package/src/commands/facilitate.js

@@ -1,6 +1,7 @@
 import { readFileSync, createWriteStream } from "node:fs";
 import { resolve } from "node:path";
 import { createFacilitator } from "../facilitator.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 
 /**
@@ -62,6 +63,11 @@ function parseFacilitateOptions(values) {
 export async function runFacilitateCommand(values, _args) {
   const opts = parseFacilitateOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   const fileStream = opts.outputPath
     ? createWriteStream(opts.outputPath)
     : null;
@@ -87,6 +93,7 @@ export async function runFacilitateCommand(values, _args) {
     maxTurns: opts.maxTurns,
     facilitatorProfile: opts.facilitatorProfile,
     taskAmend: opts.taskAmend,
+    redactor,
   });
 
   const result = await facilitator.run(opts.taskContent);