@forwardimpact/libharness 0.1.22 → 1.1.0

package/src/benchmark/scheduler.js

@@ -0,0 +1,78 @@
+/**
+ * CellScheduler — bounded concurrent execution of benchmark cells.
+ *
+ * Keeps at most `concurrency` `runCell(cell)` calls in flight at once and
+ * yields each settled record in **completion order** (not grid order). The
+ * runner's drain loop consumes this async iterable as the sole writer of
+ * `results.jsonl`, so concurrency lives here in execution while the ledger
+ * stays single-writer — no write mutex on the hot path.
+ */
+
+/** Bounded pool that streams settled cell records in completion order. */
+export class CellScheduler {
+  /**
+   * @param {object} opts
+   * @param {number} opts.concurrency - Max cells in flight (integer ≥ 1).
+   * @param {(cell: {task: object, runIndex: number}) => Promise<object>} opts.runCell -
+   *   Runs one cell to a settled record. By contract `runCell` never rejects —
+   *   the runner's `#runOne` catches setup, agent, and schema failures and
+   *   returns a record rather than throwing — but a rejection is still guarded
+   *   so one bad cell cannot wedge the drain.
+   */
+  constructor({ concurrency, runCell }) {
+    if (!Number.isInteger(concurrency) || concurrency < 1)
+      throw new Error("concurrency must be an integer ≥ 1");
+    if (typeof runCell !== "function")
+      throw new Error("runCell must be a function");
+    this.concurrency = concurrency;
+    this.runCell = runCell;
+  }
+
+  /**
+   * Run every cell with bounded concurrency, yielding each settled record the
+   * moment its cell completes.
+   * @param {{task: object, runIndex: number}[]} cells
+   * @returns {AsyncGenerator<object>}
+   */
+  async *run(cells) {
+    let next = 0;
+    /** @type {Set<Promise<{p: Promise<*>, record: object}>>} */
+    const inFlight = new Set();
+
+    const launch = () => {
+      const cell = cells[next++];
+      // The wrapper resolves to its own handle (for O(1) removal) plus the
+      // settled record, and never rejects — a thrown runCell becomes a fail
+      // record so the drain keeps consuming.
+      const p = Promise.resolve()
+        .then(() => this.runCell(cell))
+        .then(
+          (record) => ({ p, record }),
+          (error) => ({ p, record: schedulerFailRecord(cell, error) }),
+        );
+      inFlight.add(p);
+    };
+
+    while (next < cells.length && inFlight.size < this.concurrency) launch();
+    while (inFlight.size > 0) {
+      const { p, record } = await Promise.race(inFlight);
+      inFlight.delete(p);
+      yield record;
+      if (next < cells.length) launch();
+    }
+  }
+}
+
+/**
+ * Defensive fallback when `runCell` rejects (contract says it cannot). Keeps
+ * the drain consumable; the record is intentionally minimal and will be
+ * skipped by `report`'s schema validation, counted as skipped.
+ */
+function schedulerFailRecord(cell, error) {
+  return {
+    taskId: cell.task?.id,
+    runIndex: cell.runIndex,
+    verdict: "fail",
+    schedulerError: error?.message ?? String(error),
+  };
+}

package/src/benchmark/task-family.js

@@ -0,0 +1,260 @@
+/**
+ * Task-family loader. A task family is a directory under
+ *   <root>/
+ *     apm.lock.yaml
+ *     .claude/                # pre-staged skills + agents
+ *     tasks/<task_name>/
+ *       agent.task.md
+ *       supervisor.task.md    # optional; appended to the task as supervisor context
+ *       judge.task.md
+ *       hooks/                # harness-only; never copied to agent CWD
+ *         preflight.sh
+ *         invariants.sh
+ *       specs/                # copied into agent CWD
+ *       workdir/              # copied into agent CWD
+ *
+ * Local paths or git URLs are both accepted; git URLs are shallow-cloned into
+ * a temp dir and `familyRevision` becomes `git:<sha>` of HEAD at clone time.
+ * Local paths use the canonical-tree algorithm from design § Family revision
+ * algorithm so the result is stable across operating systems.
+ *
+ * Filesystem and subprocess access route through the injected `runtime` bag
+ * (`runtime.fs` async, `runtime.subprocess.run` one-shot, `tmpdir` derived
+ * from `runtime.proc.env`).
+ */
+
+import { createHash } from "node:crypto";
+import { join, posix, relative, resolve, sep } from "node:path";
+
+const GIT_URL_RE = /^(git@|https?:\/\/|ssh:\/\/|git:\/\/)/;
+const SKIP_DIRS = new Set([".git", "node_modules"]);
+// POSIX `X_OK` (execute permission); node's fs honours the numeric mode, so we
+// avoid importing `node:fs`'s `constants` (which would light the fs smell).
+const X_OK = 1;
+
+/**
+ * Derive the system temp dir from the env (node's `os.tmpdir()` is itself an
+ * env-respecting wrapper). The runtime bag has no `os` slot by design.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {string}
+ */
+function tmpdir(runtime) {
+  return runtime.proc.env.TMPDIR ?? "/tmp";
+}
+
+/**
+ * Load a task family from a local path or git URL.
+ * @param {string} rootPathOrGitUrl
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {Promise<TaskFamily>}
+ */
+export async function loadTaskFamily(rootPathOrGitUrl, runtime) {
+  if (!runtime) throw new Error("runtime is required");
+  const isGit = GIT_URL_RE.test(rootPathOrGitUrl);
+  let rootPath;
+  let familyRevision;
+  if (isGit) {
+    const dir = await runtime.fs.mkdtemp(
+      join(tmpdir(runtime), "fit-benchmark-family-"),
+    );
+    await gitClone(runtime, rootPathOrGitUrl, dir);
+    rootPath = dir;
+    familyRevision = "git:" + (await gitHead(runtime, dir));
+  } else {
+    rootPath = resolve(rootPathOrGitUrl);
+    familyRevision = "sha256:" + (await canonicalTreeHash(runtime, rootPath));
+  }
+
+  const tasks = await discoverTasks(runtime, rootPath);
+
+  return {
+    rootPath,
+    familyRevision,
+    tasks() {
+      return tasks;
+    },
+  };
+}
+
+/**
+ * Assert that `<judgeProfilesDir>/<judgeProfile>.md` exists. Called from
+ * `BenchmarkRunner.run()` so a missing judge profile fails the family
+ * install before any agent session starts.
+ * @param {TaskFamily} _family
+ * @param {string} judgeProfilesDir
+ * @param {string} judgeProfile
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {Promise<void>}
+ */
+export async function assertJudgeProfileStaged(
+  _family,
+  judgeProfilesDir,
+  judgeProfile,
+  runtime,
+) {
+  const candidate = join(judgeProfilesDir, `${judgeProfile}.md`);
+  try {
+    await runtime.fs.access(candidate);
+  } catch {
+    throw new Error(`judge profile not staged: ${candidate}`);
+  }
+}
+
+async function discoverTasks(runtime, rootPath) {
+  const fs = runtime.fs;
+  const tasksRoot = join(rootPath, "tasks");
+  const tasks = [];
+  let entries;
+  try {
+    entries = await fs.readdir(tasksRoot, { withFileTypes: true });
+  } catch (e) {
+    if (e.code === "ENOENT") return tasks;
+    throw e;
+  }
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const taskDir = join(tasksRoot, entry.name);
+    const supervisorPath = join(taskDir, "supervisor.task.md");
+    const judgePath = join(taskDir, "judge.task.md");
+    const preflightPath = join(taskDir, "hooks", "preflight.sh");
+    const invariantsPath = join(taskDir, "hooks", "invariants.sh");
+    tasks.push({
+      id: entry.name,
+      paths: {
+        taskDir,
+        instructions: join(taskDir, "agent.task.md"),
+        supervisor: (await fileExists(fs, supervisorPath))
+          ? supervisorPath
+          : null,
+        judge: (await fileExists(fs, judgePath)) ? judgePath : null,
+        hooks: join(taskDir, "hooks"),
+        preflight: (await fileExecutable(fs, preflightPath))
+          ? preflightPath
+          : null,
+        invariants: (await fileExecutable(fs, invariantsPath))
+          ? invariantsPath
+          : null,
+        specs: join(taskDir, "specs"),
+        workdir: join(taskDir, "workdir"),
+      },
+    });
+  }
+  tasks.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return tasks;
+}
+
+async function fileExists(fs, path) {
+  try {
+    await fs.access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function fileExecutable(fs, path) {
+  try {
+    await fs.access(path, X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Canonical-tree hash per design § Family revision algorithm:
+ *   list regular files (excluding .git/, node_modules/)
+ *   resolve symlinks before reading
+ *   sort by NFC-normalised POSIX-style root-relative path
+ *   row = <rel-path>\0<hex-sha256>\n
+ *   sha256(concat(rows))
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} rootPath
+ * @returns {Promise<string>} hex digest
+ */
+async function canonicalTreeHash(runtime, rootPath) {
+  const fs = runtime.fs;
+  const real = await fs.realpath(rootPath);
+  const rows = [];
+  for await (const filePath of walkFiles(fs, real)) {
+    const rel = toPosix(relative(real, filePath)).normalize("NFC");
+    const target = await fs.realpath(filePath);
+    const bytes = await fs.readFile(target);
+    const hex = createHash("sha256").update(bytes).digest("hex");
+    rows.push({ rel, hex });
+  }
+  rows.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
+  const acc = createHash("sha256");
+  for (const r of rows) acc.update(`${r.rel}\0${r.hex}\n`, "utf8");
+  return acc.digest("hex");
+}
+
+async function* walkFiles(fs, dir) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      yield* walkFiles(fs, full);
+    } else if (entry.isSymbolicLink()) {
+      const resolvedFile = await resolveSymlinkToFile(fs, full);
+      if (resolvedFile) yield full;
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+/**
+ * Return the resolved path if `linkPath` is a symlink to a regular file.
+ * Returns null for dangling symlinks or links to non-file targets.
+ */
+async function resolveSymlinkToFile(fs, linkPath) {
+  const st = await fs.lstat(linkPath);
+  if (!st.isSymbolicLink()) return null;
+  try {
+    const resolved = await fs.realpath(linkPath);
+    const tstat = await fs.lstat(resolved);
+    return tstat.isFile() ? resolved : null;
+  } catch {
+    return null;
+  }
+}
+
+function toPosix(p) {
+  if (sep === posix.sep) return p;
+  return p.split(sep).join(posix.sep);
+}
+
+async function gitClone(runtime, url, dir) {
+  await git(runtime, ["clone", "--depth", "1", url, dir]);
+}
+
+async function gitHead(runtime, dir) {
+  const out = await git(runtime, ["-C", dir, "rev-parse", "HEAD"]);
+  return out.trim();
+}
+
+async function git(runtime, args) {
+  const { stdout, stderr, exitCode } = await runtime.subprocess.run(
+    "git",
+    args,
+  );
+  if (exitCode !== 0) {
+    throw new Error(`git ${args.join(" ")} exited ${exitCode}: ${stderr}`);
+  }
+  return stdout;
+}
+
+/**
+ * @typedef {object} Task
+ * @property {string} id - Task name (directory name under tasks/)
+ * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, specs: string, workdir: string}} paths
+ */
+
+/**
+ * @typedef {object} TaskFamily
+ * @property {string} rootPath
+ * @property {string} familyRevision - `git:<sha>` or `sha256:<hex>`
+ * @property {() => Task[]} tasks
+ */

package/src/benchmark/workdir.js

@@ -0,0 +1,344 @@
+/**
+ * WorkdirManager — per-task lifecycle: create the agent CWD, seed it from the
+ * task's workdir + specs + staged .claude/, allocate a free TCP port, run
+ * the pre-flight smoke probe, and tear down the process group at end of run.
+ *
+ * The Workdir handle threads `cwd`, `port`, `pgid`, and trace paths through
+ * runAgent → invariants → judge → teardown.
+ *
+ * Filesystem, subprocess, clock, and process-signal access all route through
+ * the injected `runtime` bag. Only raw TCP plumbing (`node:net`) stays direct —
+ * it is not an ambient-dependency smell and the runtime bag models no socket
+ * surface.
+ */
+
+import { createServer } from "node:net";
+import { connect } from "node:net";
+import { join } from "node:path";
+
+import { loadEnv } from "./env-loader.js";
+import { buildHookEnv } from "./hook-env.js";
+
+const DEFAULT_TERM_GRACE_MS = 5_000;
+
+/**
+ * @typedef {object} Workdir
+ * @property {string} cwd - Agent CWD (per-task copy).
+ * @property {string} runDir - Parent of `cwd`; holds trace/log siblings.
+ * @property {number} port - Allocated TCP port for the agent.
+ * @property {number} pgid - Process-group id captured from the preflight child.
+ * @property {*} scaffold - Reserved per design § Components; v1 sets null.
+ * @property {string} agentTracePath
+ * @property {string} supervisorTracePath
+ * @property {string} judgeTracePath
+ * @property {string[]} [envNames] - Env var names loaded from .env files.
+ * @property {{phase: string, message: string, exitCode: number}} [preflightError]
+ */
+
+/** Per-task workdir lifecycle: seed → preflight → teardown. */
+export class WorkdirManager {
+  /**
+   * @param {object} deps
+   * @param {string} deps.stagingDir - Output of `installApm(...)`.
+   * @param {string} deps.runOutputDir - Root run-output directory (parent of `runs/`).
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `fs`, `subprocess`, `clock`, `proc`.
+   */
+  constructor({
+    stagingDir,
+    runOutputDir,
+    termGraceMs,
+    familyRootPath,
+    runtime,
+  }) {
+    if (!stagingDir) throw new Error("stagingDir is required");
+    if (!runOutputDir) throw new Error("runOutputDir is required");
+    if (!runtime) throw new Error("runtime is required");
+    this.stagingDir = stagingDir;
+    this.runOutputDir = runOutputDir;
+    this.termGraceMs = termGraceMs ?? DEFAULT_TERM_GRACE_MS;
+    this.familyRootPath = familyRootPath ?? null;
+    this.runtime = runtime;
+    // One registry per manager: hands out distinct, bindable ports under a lock
+    // so concurrent cells can never be handed the same number.
+    this.ports = new PortRegistry();
+  }
+
+  /**
+   * Create the per-task working directory and run the pre-flight probe.
+   * @param {import("./task-family.js").Task} task
+   * @param {number} runIndex
+   * @returns {Promise<Workdir>}
+   */
+  async start(task, runIndex) {
+    const fs = this.runtime.fs;
+    const slug = task.id.replace("/", "__");
+    const runDir = join(this.runOutputDir, "runs", slug, String(runIndex));
+    const cwd = join(runDir, "cwd");
+    await fs.mkdir(cwd, { recursive: true });
+
+    // Family-level shared fixtures: convention-over-configuration, copied if
+    // present. They form the shared base; the per-task workdir/specs below
+    // overlay on top (fs.cp defaults to force:true, so a per-task file wins).
+    if (this.familyRootPath) {
+      await fs
+        .cp(join(this.familyRootPath, "workdir"), cwd, { recursive: true })
+        .catch((e) => {
+          if (e.code !== "ENOENT") throw e;
+        });
+      await fs
+        .cp(join(this.familyRootPath, "specs"), join(cwd, "specs"), {
+          recursive: true,
+        })
+        .catch((e) => {
+          if (e.code !== "ENOENT") throw e;
+        });
+    }
+
+    await fs.cp(task.paths.workdir, cwd, { recursive: true }).catch((e) => {
+      if (e.code !== "ENOENT") throw e;
+    });
+    await fs
+      .cp(task.paths.specs, join(cwd, "specs"), {
+        recursive: true,
+      })
+      .catch((e) => {
+        if (e.code !== "ENOENT") throw e;
+      });
+    await fs.cp(join(this.stagingDir, ".claude"), join(cwd, ".claude"), {
+      recursive: true,
+    });
+    await fs
+      .cp(join(this.stagingDir, "node_modules"), join(cwd, "node_modules"), {
+        recursive: true,
+      })
+      .catch((e) => {
+        if (e.code !== "ENOENT") throw e;
+      });
+
+    const envDirs = [
+      ...(this.familyRootPath ? [this.familyRootPath] : []),
+      ...(task.paths.taskDir ? [task.paths.taskDir] : []),
+    ];
+    const envNames =
+      envDirs.length > 0 ? await loadEnv(envDirs, cwd, this.runtime) : [];
+
+    const port = await this.ports.acquire();
+    const agentTracePath = join(runDir, "agent.ndjson");
+    const supervisorTracePath = join(runDir, "supervisor.ndjson");
+    const judgeTracePath = join(runDir, "judge.ndjson");
+
+    const preflight = task.paths.preflight
+      ? await runPreflight(this.runtime, task.paths.preflight, cwd, port, {
+          taskId: task.id,
+          taskDir: task.paths.taskDir,
+          hooksDir: task.paths.hooks,
+          familyDir: this.familyRootPath,
+        })
+      : { pgid: 0 };
+
+    return {
+      cwd,
+      runDir,
+      port,
+      pgid: preflight.pgid,
+      scaffold: null,
+      agentTracePath,
+      supervisorTracePath,
+      judgeTracePath,
+      envNames,
+      ...(preflight.error && { preflightError: preflight.error }),
+    };
+  }
+
+  /**
+   * Tear down the per-task process group: SIGTERM, wait, SIGKILL, then probe.
+   * @param {Workdir} workdir
+   * @returns {Promise<{portFree: boolean, descendants: number}>}
+   */
+  async teardown(workdir) {
+    const { proc, clock } = this.runtime;
+    if (workdir.pgid && workdir.pgid > 0) {
+      try {
+        proc.kill(-workdir.pgid, "SIGTERM");
+      } catch {
+        // Process group already gone — fine.
+      }
+      await clock.sleep(this.termGraceMs);
+      try {
+        proc.kill(-workdir.pgid, "SIGKILL");
+      } catch {
+        // Already exited.
+      }
+      // Poll briefly until the process group is empty — SIGKILL returns
+      // before the kernel finishes reaping descendants.
+      await waitFor(
+        this.runtime,
+        async () => (await countDescendants(this.runtime, workdir.pgid)) === 0,
+        2_000,
+      );
+    }
+    // Release the reservation in a finally so a throwing probe cannot leak the
+    // number from the in-use set; release after the port-free probe so a freed
+    // number can be re-handed to a waiting cell.
+    try {
+      const portFree = await isPortFree(workdir.port);
+      const descendants = await countDescendants(this.runtime, workdir.pgid);
+      return { portFree, descendants };
+    } finally {
+      this.ports.release(workdir.port);
+    }
+  }
+}
+
+/**
+ * Hands out distinct, bindable TCP ports under a lock. Replaces the bare
+ * close-then-return allocator whose allocate→bind window let two concurrent
+ * cells receive the same number.
+ *
+ * The reservation is the *number*, not a held socket — a held socket could not
+ * be bound by the agent later. `acquire` serializes through a one-slot promise
+ * chain and re-probes if the OS hands back a number already in the live in-use
+ * set, so no two in-flight cells share a port.
+ */
+export class PortRegistry {
+  #inUse = new Set();
+  #tail = Promise.resolve();
+
+  /** @returns {Promise<number>} A distinct, currently-bindable port. */
+  acquire() {
+    const next = this.#tail.then(async () => {
+      let port;
+      do {
+        port = await probeFreePort();
+      } while (this.#inUse.has(port));
+      this.#inUse.add(port);
+      return port;
+    });
+    // Keep the chain alive even if one acquire rejects, so later acquires
+    // still run; swallow here, surface the rejection on `next`.
+    this.#tail = next.catch(() => {});
+    return next;
+  }
+
+  /** @param {number} port */
+  release(port) {
+    this.#inUse.delete(port);
+  }
+}
+
+/**
+ * Spawn preflight. Stays detached so we can SIGTERM the whole process group.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} script
+ * @param {string} cwd - Agent CWD passed via $AGENT_CWD.
+ * @param {number} port - Free TCP port passed via $PORT.
+ * @param {{taskId: string, taskDir: string, hooksDir: string, familyDir: string|null}} vars - Extra hook env vars.
+ * @returns {Promise<{pgid: number, error?: {phase: string, message: string, exitCode: number}}>}
+ */
+async function runPreflight(runtime, script, cwd, port, vars) {
+  const child = runtime.subprocess.spawn(script, [], {
+    cwd,
+    env: buildHookEnv(runtime.proc.env, { cwd, port, ...vars }),
+    detached: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  if (child.pid === undefined) {
+    throw new Error(`failed to spawn preflight: ${script}`);
+  }
+  const pgid = child.pid;
+  let stderr = "";
+  const drainStdout = (async () => {
+    for await (const _chunk of child.stdout) {
+      // discard
+    }
+  })();
+  for await (const chunk of child.stderr) stderr += chunk.toString();
+  await drainStdout;
+  const code = await child.exitCode;
+  if (code === 0) return { pgid };
+  const message = stderr.trim() || `preflight exited with code ${code}`;
+  return {
+    pgid,
+    error: {
+      phase: "preflight",
+      message,
+      exitCode: typeof code === "number" ? code : -1,
+    },
+  };
+}
+
+function probeFreePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}
+
+function isPortFree(port) {
+  if (!port) return Promise.resolve(true);
+  return new Promise((res) => {
+    const socket = connect({ port, host: "127.0.0.1" }, () => {
+      socket.destroy();
+      res(false);
+    });
+    socket.on("error", () => res(true));
+    socket.setTimeout(500, () => {
+      socket.destroy();
+      res(true);
+    });
+  });
+}
+
+async function countDescendants(runtime, pgid) {
+  if (!pgid || pgid <= 0) return 0;
+  const child = runtime.subprocess.spawn(
+    "ps",
+    ["-o", "pid=", "-g", String(pgid)],
+    {
+      stdio: ["ignore", "pipe", "ignore"],
+    },
+  );
+  let out = "";
+  try {
+    for await (const chunk of child.stdout) out += chunk.toString();
+    await child.exitCode;
+  } catch {
+    return 0;
+  }
+  const pids = out
+    .split("\n")
+    .map((s) => s.trim())
+    .filter(Boolean)
+    .filter((s) => Number(s) !== runtime.proc.pid);
+  return pids.length;
+}
+
+async function waitFor(runtime, predicate, timeoutMs) {
+  const deadline = runtime.clock.now() + timeoutMs;
+  while (runtime.clock.now() < deadline) {
+    if (await predicate()) return true;
+    await runtime.clock.sleep(50);
+  }
+  return false;
+}
+
+/**
+ * Factory function — wires real dependencies.
+ * @param {ConstructorParameters<typeof WorkdirManager>[0]} deps
+ * @returns {WorkdirManager}
+ */
+export function createWorkdirManager(deps) {
+  return new WorkdirManager(deps);
+}