@forwardimpact/libeval 0.1.39 → 0.1.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.39",
+  "version": "0.1.42",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/agent-runner.js

@@ -77,22 +77,7 @@ export class AgentRunner {
     try {
       const iterator = this.query({
         prompt: effectiveTask,
-        options: {
-          cwd: this.cwd,
-          allowedTools: this.allowedTools,
-          maxTurns:
-            this.maxTurns === 0 ? Number.MAX_SAFE_INTEGER : this.maxTurns,
-          model: this.model,
-          permissionMode: PERMISSION_MODE,
-          allowDangerouslySkipPermissions: true,
-          settingSources: this.settingSources,
-          abortController,
-          ...(this.disallowedTools.length > 0 && {
-            disallowedTools: this.disallowedTools,
-          }),
-          ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
-          ...(this.mcpServers && { mcpServers: this.mcpServers }),
-        },
+        options: this.#callOptions(abortController),
       });
       return await this.#consumeQuery(iterator);
     } finally {
@@ -112,12 +97,8 @@ export class AgentRunner {
       const iterator = this.query({
         prompt,
         options: {
+          ...this.#callOptions(abortController),
           resume: this.sessionId,
-          model: this.model,
-          permissionMode: PERMISSION_MODE,
-          allowDangerouslySkipPermissions: true,
-          abortController,
-          ...(this.mcpServers && { mcpServers: this.mcpServers }),
         },
       });
       return await this.#consumeQuery(iterator);
@@ -126,6 +107,37 @@ export class AgentRunner {
     }
   }
 
+  /**
+   * Build the options passed to every SDK query() call. Shared by run() and
+   * resume() so the agent's configuration — cwd, tools, prompt, setting
+   * sources, turn budget — is identical across the session's lifetime. Only
+   * resume() layers `resume: this.sessionId` on top.
+   *
+   * SDK options are call-attached, not session-attached: the resumed call
+   * loads the prior conversation but otherwise uses whatever options this
+   * call passes. Omitting tool/prompt/setting options on resume causes the
+   * agent to silently lose its restrictions and persona between turns.
+   * @param {AbortController} abortController
+   * @returns {object}
+   */
+  #callOptions(abortController) {
+    return {
+      cwd: this.cwd,
+      allowedTools: this.allowedTools,
+      maxTurns: this.maxTurns === 0 ? Number.MAX_SAFE_INTEGER : this.maxTurns,
+      model: this.model,
+      permissionMode: PERMISSION_MODE,
+      allowDangerouslySkipPermissions: true,
+      settingSources: this.settingSources,
+      abortController,
+      ...(this.disallowedTools.length > 0 && {
+        disallowedTools: this.disallowedTools,
+      }),
+      ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
+      ...(this.mcpServers && { mcpServers: this.mcpServers }),
+    };
+  }
+
   /**
    * Shared consumer for both `run()` and `resume()`. Iterates the SDK query
    * iterator, mirroring every line to the output stream / buffer / onLine

package/src/benchmark/apm-installer.js

@@ -3,71 +3,109 @@
  * materialise skills and agents, copies the resulting `.claude/` into a
  * staging directory, and computes the manifest fingerprint from the lockfile.
  * Per-task copy happens later in WorkdirManager.
+ *
+ * The class takes a `spawn` seam so tests can substitute a fake child process
+ * without ever shelling out to a real `apm` binary. See `createApmInstaller`
+ * for the real-dependency wiring; `installApm` is a thin free-function wrapper
+ * for callers that don't need to inject anything.
  */
 
-import { spawn } from "node:child_process";
+import { spawn as nodeSpawn } from "node:child_process";
 import { createHash } from "node:crypto";
 import { access, cp, mkdir, readFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 
-/**
- * @param {import("./task-family.js").TaskFamily} family
- * @param {string} outputDir - The benchmark run's output directory.
- * @returns {Promise<{stagingDir: string, skillSetHash: string}>}
- */
-export async function installApm(family, outputDir) {
-  const stagingDir = join(outputDir, ".apm-staging");
-  const stagedClaude = join(stagingDir, ".claude");
-  const sourceClaude = join(family.rootPath, ".claude");
-  const apmYml = join(family.rootPath, "apm.yml");
+/** Installs apm and stages `.claude/` for a task family. */
+export class ApmInstaller {
+  /**
+   * @param {object} [deps]
+   * @param {typeof nodeSpawn} [deps.spawn] - Spawn seam (defaults to
+   *   `node:child_process` spawn). Tests inject a fake to avoid shelling out.
+   */
+  constructor({ spawn } = {}) {
+    this.spawn = spawn ?? nodeSpawn;
+  }
 
-  const hasApm = await access(apmYml)
-    .then(() => true)
-    .catch(() => false);
+  /**
+   * @param {import("./task-family.js").TaskFamily} family
+   * @param {string} outputDir - The benchmark run's output directory.
+   * @returns {Promise<{stagingDir: string, skillSetHash: string, judgeProfilesDir: string}>}
+   */
+  async install(family, outputDir) {
+    const stagingDir = join(outputDir, ".apm-staging");
+    const stagedClaude = join(stagingDir, ".claude");
+    const sourceClaude = join(family.rootPath, ".claude");
+    const apmYml = join(family.rootPath, "apm.yml");
+
+    const hasApm = await access(apmYml)
+      .then(() => true)
+      .catch(() => false);
+
+    if (hasApm) {
+      await this.#runApmInstall(family.rootPath);
+      try {
+        await access(sourceClaude);
+      } catch {
+        throw new Error(
+          `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
+        );
+      }
+    }
 
-  if (hasApm) {
-    await runApmInstall(family.rootPath);
+    await rm(stagingDir, { recursive: true, force: true });
+    const hasClaudeDir = await access(sourceClaude)
+      .then(() => true)
+      .catch(() => false);
+    if (hasClaudeDir) {
+      await cp(sourceClaude, stagedClaude, { recursive: true });
+    } else {
+      await mkdir(stagedClaude, { recursive: true });
+    }
+
+    // Stage the family-local judge profile outside .claude/ so it is available
+    // to the judge but never copied into the agent-under-test's CWD.
+    const judgeSource = join(family.rootPath, "judge.md");
+    const judgeProfilesDir = join(stagingDir, "judge-profiles");
+    try {
+      await access(judgeSource);
+      await mkdir(judgeProfilesDir, { recursive: true });
+      await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
+    } catch {}
+
+    const lockPath = join(family.rootPath, "apm.lock.yaml");
+    let skillSetHash = "";
     try {
-      await access(sourceClaude);
+      const lockBytes = await readFile(lockPath);
+      skillSetHash =
+        "sha256:" +
+        createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
     } catch {
-      throw new Error(
-        `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
-      );
+      // No lockfile — family doesn't use skill packs.
     }
-  }
 
-  await rm(stagingDir, { recursive: true, force: true });
-  const hasClaudeDir = await access(sourceClaude)
-    .then(() => true)
-    .catch(() => false);
-  if (hasClaudeDir) {
-    await cp(sourceClaude, stagedClaude, { recursive: true });
-  } else {
-    await mkdir(stagedClaude, { recursive: true });
+    return { stagingDir, skillSetHash, judgeProfilesDir };
   }
 
-  // Stage the family-local judge profile outside .claude/ so it is available
-  // to the judge but never copied into the agent-under-test's CWD.
-  const judgeSource = join(family.rootPath, "judge.md");
-  const judgeProfilesDir = join(stagingDir, "judge-profiles");
-  try {
-    await access(judgeSource);
-    await mkdir(judgeProfilesDir, { recursive: true });
-    await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
-  } catch {}
-
-  const lockPath = join(family.rootPath, "apm.lock.yaml");
-  let skillSetHash = "";
-  try {
-    const lockBytes = await readFile(lockPath);
-    skillSetHash =
-      "sha256:" +
-      createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
-  } catch {
-    // No lockfile — family doesn't use skill packs.
+  #runApmInstall(cwd) {
+    return new Promise((res, rej) => {
+      const child = this.spawn("apm", ["install", "--target", "claude"], {
+        cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      let stderr = "";
+      child.stdout.on("data", () => {});
+      child.stderr.on("data", (d) => {
+        stderr += d.toString();
+      });
+      child.on("error", (e) => {
+        rej(new Error(`failed to spawn apm: ${e.message}`));
+      });
+      child.on("close", (code) => {
+        if (code === 0) res();
+        else rej(new Error(`apm install exited ${code}: ${stderr}`));
+      });
+    });
   }
-
-  return { stagingDir, skillSetHash, judgeProfilesDir };
 }
 
 function normalizeLf(buf) {
@@ -79,23 +117,20 @@ function normalizeLf(buf) {
   return Buffer.from(out);
 }
 
-function runApmInstall(cwd) {
-  return new Promise((res, rej) => {
-    const child = spawn("apm", ["install", "--target", "claude"], {
-      cwd,
-      stdio: ["ignore", "pipe", "pipe"],
-    });
-    let stderr = "";
-    child.stdout.on("data", () => {});
-    child.stderr.on("data", (d) => {
-      stderr += d.toString();
-    });
-    child.on("error", (e) => {
-      rej(new Error(`failed to spawn apm: ${e.message}`));
-    });
-    child.on("close", (code) => {
-      if (code === 0) res();
-      else rej(new Error(`apm install exited ${code}: ${stderr}`));
-    });
-  });
+/**
+ * Factory function — wires real dependencies.
+ * @param {ConstructorParameters<typeof ApmInstaller>[0]} [deps]
+ * @returns {ApmInstaller}
+ */
+export function createApmInstaller(deps) {
+  return new ApmInstaller(deps);
+}
+
+/**
+ * Free-function shorthand for callers that don't need to inject a spawn seam.
+ * @param {import("./task-family.js").TaskFamily} family
+ * @param {string} outputDir
+ */
+export function installApm(family, outputDir) {
+  return new ApmInstaller().install(family, outputDir);
 }

package/src/benchmark/runner.js

@@ -21,7 +21,7 @@ import { join, resolve as resolvePath } from "node:path";
 
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
 import { createSupervisor } from "../supervisor.js";
-import { installApm } from "./apm-installer.js";
+import { installApm as defaultInstallApm } from "./apm-installer.js";
 import { runJudge } from "./judge.js";
 import { validateResultRecord } from "./result.js";
 import { runScoring } from "./scorer.js";
@@ -64,6 +64,10 @@ export class BenchmarkRunner {
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
    *   contract as `runJudge(task, workdir, scoring, deps)`. Internal testing
    *   only.
+   * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
+   *   Same contract as `installApm(family, outputDir)`. Lets tests inject a
+   *   fake `apm` spawn (or skip the install entirely) so the suite never
+   *   shells out to a real `apm` binary. Internal testing only.
    */
   constructor({
     family,
@@ -81,6 +85,7 @@ export class BenchmarkRunner {
     runAgent,
     runScoring: runScoringHook,
     runJudge: runJudgeHook,
+    installApm: installApmHook,
   }) {
     if (!family) throw new Error("family is required");
     if (!Number.isInteger(runs) || runs < 1)
@@ -105,6 +110,7 @@ export class BenchmarkRunner {
     this._runAgentHook = runAgent ?? null;
     this._runScoringHook = runScoringHook ?? runScoring;
     this._runJudgeHook = runJudgeHook ?? runJudge;
+    this._installApmHook = installApmHook ?? defaultInstallApm;
   }
 
   /**
@@ -118,10 +124,8 @@ export class BenchmarkRunner {
         : this.familyInput;
 
     await mkdir(this.output, { recursive: true });
-    const { stagingDir, skillSetHash, judgeProfilesDir } = await installApm(
-      family,
-      this.output,
-    );
+    const { stagingDir, skillSetHash, judgeProfilesDir } =
+      await this._installApmHook(family, this.output);
 
     const tasks = family.tasks();
     if (this.profiles.judge) {

package/src/facilitator.js

@@ -394,6 +394,8 @@ const devNull = new Writable({
  * @param {string} [deps.agentModel] - Agent model override (falls back to `model`).
  * @param {string} [deps.facilitatorModel] - Facilitator model override (falls back to `model`).
  * @param {number} [deps.maxTurns]
+ * @param {string[]} [deps.facilitatorAllowedTools] - Tools the facilitator may use; defaults to a read/write file-edit set.
+ * @param {string[]} [deps.facilitatorDisallowedTools] - Additional tools to block on the facilitator; merged with the sub-agent spawn defaults (Agent/Task/TaskOutput/TaskStop).
  * @param {string} [deps.facilitatorProfile] - Facilitator profile name; resolved into the main-thread system prompt via `composeProfilePrompt`.
  * @param {string} [deps.profilesDir] - Directory containing `<name>.md` profile files. Defaults to `<facilitatorCwd>/.claude/agents`. Resolved once from the facilitator's cwd so profiles travel with the project, not with per-agent sandboxes.
  * @param {string} [deps.taskAmend] - Opaque addendum appended to the task before delivery.
@@ -408,6 +410,8 @@ export function createFacilitator({
   agentModel,
   facilitatorModel,
   maxTurns,
+  facilitatorAllowedTools,
+  facilitatorDisallowedTools,
   facilitatorProfile,
   profilesDir,
   taskAmend,
@@ -467,12 +471,29 @@ export function createFacilitator({
     return { name: config.name, role: config.role, runner };
   });
 
+  // Block the SDK's sub-agent spawn tools on the facilitator: its job is to
+  // coordinate participants through the libeval orchestration harness, not
+  // to fan work out to ad-hoc Claude Code sub-agents. Mirrors the supervisor.
+  const defaultDisallowed = ["Agent", "Task", "TaskOutput", "TaskStop"];
+  const disallowedTools = facilitatorDisallowedTools
+    ? [...new Set([...defaultDisallowed, ...facilitatorDisallowedTools])]
+    : defaultDisallowed;
+
   const facilitatorRunner = createAgentRunner({
     cwd: facilitatorCwd,
     query,
     output: devNull,
     model: facilitatorModel ?? model,
     maxTurns: maxTurns ?? 20,
+    allowedTools: facilitatorAllowedTools ?? [
+      "Bash",
+      "Read",
+      "Glob",
+      "Grep",
+      "Write",
+      "Edit",
+    ],
+    disallowedTools,
     onLine: (line) => facilitator.emitLine("facilitator", line),
     mcpServers: { orchestration: facilitatorServer },
     settingSources: ["project"],

package/src/supervisor.js

@@ -104,7 +104,6 @@ export class Supervisor {
    */
   async run(task) {
     const initialTask = this.taskAmend ? `${task}\n\n${this.taskAmend}` : task;
-    this.taskContext = initialTask;
     this.currentSource = "supervisor";
     this.currentTurn = 0;
     let supervisorResult = await this.supervisorRunner.run(initialTask);
@@ -252,22 +251,6 @@ export class Supervisor {
     return { type: "continue" };
   }
 
-  /**
-   * Resume the supervisor runner, falling back to a fresh session when the
-   * SDK reports that the conversation no longer exists (e.g. session GC'd
-   * while the agent was running). The fresh session includes the original
-   * task context so the supervisor can still evaluate the agent's work.
-   * @param {string} prompt
-   * @returns {Promise<object>}
-   */
-  async #resumeSupervisor(prompt) {
-    const result = await this.supervisorRunner.resume(prompt);
-    if (result.error && isSessionNotFound(result.error)) {
-      return this.supervisorRunner.run(`${this.taskContext}\n\n${prompt}`);
-    }
-    return result;
-  }
-
   /**
    * If the agent has an unanswered ask, drain reminders and return a
    * formatted relay string. Returns null when no relay is needed.
@@ -295,7 +278,7 @@ export class Supervisor {
     this.currentSource = "supervisor";
     this.ctx.redirect = null;
 
-    await this.#resumeSupervisor(
+    await this.supervisorRunner.resume(
       `The agent is mid-turn. Latest batch:\n\n${batchTranscript}\n\n` +
         `Review and use your tools if action is needed.`,
     );
@@ -333,7 +316,7 @@ export class Supervisor {
           `Review and decide how to proceed.`
         : `The agent reported:\n\n${agentTranscript}\n\nReview the agent's work and decide how to proceed.`;
 
-    let supervisorResult = await this.#resumeSupervisor(reviewPrompt);
+    let supervisorResult = await this.supervisorRunner.resume(reviewPrompt);
 
     if (supervisorResult.error) {
       this.emitSummary({ success: false, turns: turn });
@@ -354,7 +337,7 @@ export class Supervisor {
     if (this.#checkAsk("supervisor") === "recheck" && !this.ctx.concluded) {
       const reminders = this.messageBus.drain("supervisor");
       if (reminders.length > 0) {
-        supervisorResult = await this.#resumeSupervisor(
+        supervisorResult = await this.supervisorRunner.resume(
           formatMessages(reminders),
         );
         if (this.ctx.concluded) {
@@ -578,6 +561,9 @@ export function createSupervisor({
     redactor,
   });
 
+  // Block the SDK's sub-agent spawn tools on the supervisor: its job is to
+  // coordinate the agent through the libeval orchestration harness, not to
+  // fan work out to ad-hoc Claude Code sub-agents. Mirrors the facilitator.
   const defaultDisallowed = ["Agent", "Task", "TaskOutput", "TaskStop"];
   const disallowedTools = supervisorDisallowedTools
     ? [...new Set([...defaultDisallowed, ...supervisorDisallowedTools])]
@@ -617,8 +603,3 @@ export function createSupervisor({
   });
   return supervisor;
 }
-
-function isSessionNotFound(error) {
-  const msg = error?.message ?? String(error);
-  return msg.includes("No conversation found with session ID");
-}