@forwardimpact/libharness 0.1.20 → 1.0.0

package/src/benchmark/invariants.js

@@ -0,0 +1,141 @@
+/**
+ * Invariants — runs `<task.paths.hooks>/invariants.sh` from the template path
+ * against the post-run agent CWD. The exit code is authoritative for the
+ * verdict; structured per-check rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON.
+ *
+ * Subprocess access flows through `runtime.subprocess.spawn`; the fd-3 backing
+ * store and the stderr log use the sync filesystem surface (`runtime.fsSync`) —
+ * the only surface this module touches, per design Decision 7.
+ */
+
+import { join } from "node:path";
+
+import { buildHookEnv } from "./hook-env.js";
+
+/**
+ * @typedef {object} InvariantsResult
+ * @property {"pass" | "fail"} verdict
+ * @property {Array<object>} details
+ * @property {number} exitCode
+ * @property {string} [stderr] - Trimmed script stderr, present only when the
+ *   script wrote to stderr. Surfaces hook failures (e.g. a missing tool) that
+ *   leave `details` empty, so they read distinctly from a real invariant miss.
+ */
+
+/**
+ * Run the task's invariants script.
+ * @param {import("./task-family.js").Task} task
+ * @param {{cwd: string, port: number, runDir: string, familyDir?: string|null}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {Promise<InvariantsResult>}
+ */
+export async function runInvariants(task, ctx, runtime) {
+  if (!runtime) throw new Error("runtime is required");
+  if (!task.paths.invariants) {
+    return { verdict: "pass", details: [], exitCode: 0 };
+  }
+  const fsSync = runtime.fsSync;
+  const script = task.paths.invariants;
+  const stderrLogPath = join(ctx.runDir, "invariants.stderr.log");
+
+  // Bun's child_process pipe setup for fd >= 3 is racy under load (it
+  // creates a unix socket pair and the connect() can return ENOENT). Use
+  // a temp file as the fd-3 backing store instead — the script still
+  // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
+  const fd3Path = join(ctx.runDir, "invariants.fd3.ndjson");
+  const fd3File = fsSync.openSync(fd3Path, "w+");
+
+  let child;
+  try {
+    child = runtime.subprocess.spawn(script, [], {
+      env: {
+        ...buildHookEnv(runtime.proc.env, {
+          cwd: ctx.cwd,
+          port: ctx.port,
+          taskId: task.id,
+          taskDir: task.paths.taskDir,
+          hooksDir: task.paths.hooks,
+          familyDir: ctx.familyDir,
+        }),
+        RESULTS_FD: "3",
+      },
+      stdio: ["inherit", "pipe", "pipe", fd3File],
+    });
+  } catch (e) {
+    tryClose(fsSync, fd3File);
+    throw e;
+  }
+
+  // Drain stdout (do not require consumers to read it); capture stderr to log.
+  const drainStdout = (async () => {
+    for await (const _chunk of child.stdout) {
+      // discard
+    }
+  })();
+  let stderr = "";
+  for await (const chunk of child.stderr) stderr += chunk.toString();
+  await drainStdout;
+  const code = await child.exitCode;
+
+  fsSync.writeFileSync(stderrLogPath, stderr);
+  tryClose(fsSync, fd3File);
+
+  const raw = readAndUnlink(fsSync, fd3Path);
+  const details = [];
+  parseFd3Buffer(raw, details);
+  const exitCode = typeof code === "number" ? code : -1;
+  const result = {
+    verdict: exitCode === 0 ? "pass" : "fail",
+    details,
+    exitCode,
+  };
+  const trimmedStderr = stderr.trim();
+  if (trimmedStderr) result.stderr = trimmedStderr;
+  return result;
+}
+
+function pushRow(line, details) {
+  const trimmed = line.trim();
+  if (!trimmed) return;
+  try {
+    details.push(JSON.parse(trimmed));
+  } catch {
+    details.push({ raw: trimmed, parseError: true });
+  }
+}
+
+function tryClose(fsSync, fd) {
+  try {
+    fsSync.closeSync(fd);
+  } catch {
+    // already closed
+  }
+}
+
+function readAndUnlink(fsSync, path) {
+  let raw = "";
+  try {
+    raw = fsSync.readFileSync(path, "utf8");
+  } catch {
+    // empty
+  }
+  try {
+    fsSync.unlinkSync(path);
+  } catch {
+    // best-effort cleanup
+  }
+  return raw;
+}
+
+/**
+ * Parse the fd-3 buffer (read from the temp-file backing) into one NDJSON
+ * row per detail entry.
+ */
+function parseFd3Buffer(buf, details) {
+  if (!buf) return;
+  const parts = buf.split("\n");
+  for (let i = 0; i < parts.length - 1; i++) pushRow(parts[i], details);
+  if (parts[parts.length - 1].trim()) {
+    pushRow(parts[parts.length - 1], details);
+  }
+}

package/src/benchmark/judge.js

@@ -0,0 +1,187 @@
+/**
+ * Benchmark adapter for the libharness `Judge`. Templates the family's
+ * `judge.task.md` with structured context variables, runs the judge against
+ * the post-run agent CWD, and returns the verdict in the benchmark's
+ * `pass`/`fail` vocabulary (mapped from libharness's `success`/`failure`).
+ *
+ * Template variables available in `judge.task.md`:
+ *
+ *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
+ *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
+ *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
+ *   {{INVARIANTS_RESULT}}   — JSON invariants object
+ *   {{SKILL_SET_HASH}}      — SHA-256 from apm.lock.yaml
+ *   {{TASK_ID}}             — task name (directory under tasks/)
+ *   {{TASK_DIR}}            — agent working directory path
+ *
+ * The judge verdict is captured from the orchestration context's
+ * `concluded` flag directly — no trace parsing on the happy path.
+ * `parseConcludeFromTrace` is preserved for offline analysis and as a
+ * fallback when the runtime ctx isn't available (e.g. re-grading a
+ * historical run from its judge.ndjson file).
+ */
+
+import { createJudge } from "../judge.js";
+import { createRedactor } from "../redaction.js";
+import { sumTraceCost } from "../cost.js";
+
+/**
+ * @typedef {object} JudgeVerdict
+ * @property {"pass" | "fail"} verdict
+ * @property {string} summary
+ * @property {number} costUsd - Cost of the judge's own SDK session.
+ */
+
+/**
+ * @typedef {object} JudgeContext
+ * @property {string} agentInstructions - Contents of agent.task.md.
+ * @property {string} agentProfile - Agent profile body (empty string if none).
+ * @property {string} skillSetHash - SHA-256 fingerprint from apm.lock.yaml.
+ */
+
+/**
+ * Run the judge over a completed task run.
+ * @param {import("./task-family.js").Task} task
+ * @param {import("./workdir.js").Workdir} workdir
+ * @param {import("./invariants.js").InvariantsResult} invariants
+ * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string, runtime: import("@forwardimpact/libutil/runtime").Runtime}} deps
+ * @param {JudgeContext} [context]
+ * @returns {Promise<JudgeVerdict>}
+ */
+export async function runJudge(task, workdir, invariants, deps, context) {
+  const runtime = deps.runtime;
+  if (!runtime) throw new Error("runtime is required");
+  const fs = runtime.fs;
+  const template = await fs.readFile(task.paths.judge, "utf8");
+  const invariantsJson = JSON.stringify(invariants, null, 2);
+  const taskText = template
+    .replaceAll("{{INVARIANTS_RESULT}}", invariantsJson)
+    .replaceAll("{{AGENT_TRACE_PATH}}", workdir.agentTracePath)
+    .replaceAll("{{AGENT_INSTRUCTIONS}}", context?.agentInstructions ?? "")
+    .replaceAll("{{AGENT_PROFILE}}", context?.agentProfile ?? "")
+    .replaceAll("{{SKILL_SET_HASH}}", context?.skillSetHash ?? "")
+    .replaceAll("{{TASK_ID}}", task.id)
+    .replaceAll("{{TASK_DIR}}", workdir.cwd);
+
+  const output = fs.createWriteStream(workdir.judgeTracePath);
+  const judge = createJudge({
+    cwd: workdir.cwd,
+    query: deps.query,
+    output,
+    model: deps.model,
+    judgeProfile: deps.judgeProfile,
+    profilesDir: deps.profilesDir,
+    maxTurns: 25,
+    redactor: createRedactor({ runtime }),
+    runtime,
+  });
+
+  let outcome;
+  try {
+    outcome = await judge.run(taskText);
+  } finally {
+    await new Promise((r) => output.end(r));
+  }
+
+  // The judge is its own SDK session; its spend lands in the judge trace we
+  // just wrote, not in the supervisor's combined trace. Read it back so the
+  // benchmark record's cost includes the judge.
+  const judgeTrace = await fs
+    .readFile(workdir.judgeTracePath, "utf8")
+    .catch(() => "");
+  const { totalCostUsd } = sumTraceCost(judgeTrace.split("\n"));
+
+  if (outcome.verdict === null) {
+    return {
+      verdict: "fail",
+      summary: "judge did not conclude",
+      costUsd: totalCostUsd,
+    };
+  }
+  return {
+    verdict: outcome.verdict === "success" ? "pass" : "fail",
+    summary: outcome.summary ?? "",
+    costUsd: totalCostUsd,
+  };
+}
+
+/**
+ * Parse the last judge-source (or supervisor-source, for backward compat
+ * with pre-Judge-class traces) `Conclude` tool call from an NDJSON trace
+ * and map the verdict (`success → pass`, `failure → fail`). Preserved for
+ * offline analysis; not used on the runtime happy path.
+ * @param {string} tracePath
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {Promise<JudgeVerdict | null>}
+ */
+export async function parseConcludeFromTrace(tracePath, runtime) {
+  if (!runtime) throw new Error("runtime is required");
+  const content = await runtime.fs.readFile(tracePath, "utf8");
+  let last = null;
+  for (const line of content.split("\n")) {
+    const candidate = extractConcludeInput(line);
+    if (candidate) last = candidate;
+  }
+  if (!last) return null;
+  return {
+    verdict: last.verdict === "success" ? "pass" : "fail",
+    summary: last.summary ?? "",
+  };
+}
+
+/**
+ * Return the `Conclude` tool input if the line carries a judge-source or
+ * supervisor-source assistant message ending in a `Conclude` tool_use
+ * block; null otherwise.
+ * @param {string} line
+ * @returns {{verdict: string, summary?: string} | null}
+ */
+function extractConcludeInput(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+  let event;
+  try {
+    event = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+  const wrapped =
+    event.event && typeof event.source === "string"
+      ? { source: event.source, inner: event.event }
+      : { source: null, inner: event };
+  if (
+    wrapped.source !== null &&
+    wrapped.source !== "judge" &&
+    wrapped.source !== "supervisor"
+  ) {
+    return null;
+  }
+  if (wrapped.inner.type !== "assistant") return null;
+  const content = wrapped.inner.message?.content ?? wrapped.inner.content;
+  if (!Array.isArray(content)) return null;
+  let found = null;
+  for (const block of content) {
+    if (
+      block.type === "tool_use" &&
+      isConcludeToolName(block.name) &&
+      block.input
+    ) {
+      found = block.input;
+    }
+  }
+  return found;
+}
+
+/**
+ * The Claude Agent SDK reports MCP tool names as
+ * `mcp__<server>__<tool>` when the model invokes them — the orchestration
+ * `Conclude` arrives as `mcp__orchestration__Conclude`. Pre-baked
+ * supervisor traces (and the libharness-internal envelopes) sometimes carry
+ * the bare `Conclude` name. Accept both forms so the parser is robust to
+ * trace source.
+ */
+function isConcludeToolName(name) {
+  if (typeof name !== "string") return false;
+  if (name === "Conclude") return true;
+  return name.endsWith("__Conclude");
+}

package/src/benchmark/npm-installer.js

@@ -0,0 +1,87 @@
+/**
+ * NpmInstaller — runs `bun install` in the family root when a package.json
+ * is present, then copies the resulting `node_modules/` into the staging
+ * directory so WorkdirManager can seed each per-task CWD.
+ *
+ * Symmetric to ApmInstaller: the subprocess and filesystem flow through the
+ * injected `runtime` bag (`runtime.subprocess.spawn` + `runtime.fs`).
+ */
+
+import { join } from "node:path";
+
+/** Run `bun install` in the family root and stage node_modules/ for per-task CWDs. */
+export class NpmInstaller {
+  /**
+   * @param {object} deps
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `subprocess.spawn` and `fs`.
+   */
+  constructor({ runtime }) {
+    if (!runtime) throw new Error("runtime is required");
+    this.runtime = runtime;
+  }
+
+  /**
+   * @param {import("./task-family.js").TaskFamily} family
+   * @param {string} stagingDir - The staging directory (created by ApmInstaller).
+   * @returns {Promise<void>}
+   */
+  async install(family, stagingDir) {
+    const fs = this.runtime.fs;
+    const pkgJson = join(family.rootPath, "package.json");
+    const hasPkg = await fs
+      .access(pkgJson)
+      .then(() => true)
+      .catch(() => false);
+    if (!hasPkg) return;
+
+    await this.#runBunInstall(family.rootPath);
+
+    const sourceModules = join(family.rootPath, "node_modules");
+    try {
+      await fs.access(sourceModules);
+    } catch {
+      throw new Error(
+        `bun install did not produce node_modules/ at ${sourceModules}; check the family's package.json`,
+      );
+    }
+
+    await fs.cp(sourceModules, join(stagingDir, "node_modules"), {
+      recursive: true,
+    });
+  }
+
+  async #runBunInstall(cwd) {
+    const child = this.runtime.subprocess.spawn("bun", ["install"], {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    const drainStdout = (async () => {
+      for await (const _chunk of child.stdout) {
+        // discard
+      }
+    })();
+    for await (const chunk of child.stderr) stderr += chunk.toString();
+    await drainStdout;
+    const code = await child.exitCode;
+    if (code !== 0) {
+      throw new Error(`bun install exited ${code}: ${stderr}`);
+    }
+  }
+}
+
+/** Factory function — wires real dependencies. */
+export function createNpmInstaller(deps) {
+  return new NpmInstaller(deps);
+}
+
+/**
+ * Free-function shorthand for callers that thread a runtime bag.
+ * @param {import("./task-family.js").TaskFamily} family
+ * @param {string} stagingDir
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ */
+export function installNpm(family, stagingDir, runtime) {
+  return new NpmInstaller({ runtime }).install(family, stagingDir);
+}