@forwardimpact/libeval 0.1.55 → 0.1.57

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.55",
+  "version": "0.1.57",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/benchmark/hook-env.js

@@ -0,0 +1,37 @@
+/**
+ * Shared environment builder for the benchmark hook scripts (`preflight.sh` and
+ * `invariants.sh`). Keeping both spawns on one helper guarantees they expose the
+ * same variable set, so hook authors never have to wonder which vars a given
+ * hook receives.
+ *
+ * Path vars (TASK_DIR, FAMILY_DIR, HOOKS_DIR) let hooks reference real
+ * locations instead of reconstructing them from `$0`. They are paths, not
+ * secrets, so they need no redaction allowlist entry.
+ */
+
+/**
+ * @param {Record<string, string>} baseEnv - Inherited env (`runtime.proc.env`).
+ * @param {object} vars
+ * @param {string} vars.cwd - Agent CWD → `$WORKDIR`.
+ * @param {number} vars.port - Allocated TCP port → `$PORT`.
+ * @param {string} vars.taskId - Task id → `$TASK_ID`.
+ * @param {string} vars.taskDir - Task directory on host → `$TASK_DIR`.
+ * @param {string} vars.hooksDir - Task `hooks/` dir on host → `$HOOKS_DIR`.
+ * @param {string|null} vars.familyDir - Family root on host → `$FAMILY_DIR`
+ *   (null when the family root is unknown, e.g. a standalone task).
+ * @returns {Record<string, string>}
+ */
+export function buildHookEnv(
+  baseEnv,
+  { cwd, port, taskId, taskDir, hooksDir, familyDir },
+) {
+  return {
+    ...baseEnv,
+    WORKDIR: cwd,
+    PORT: String(port),
+    TASK_ID: taskId,
+    TASK_DIR: taskDir,
+    HOOKS_DIR: hooksDir,
+    FAMILY_DIR: familyDir ?? "",
+  };
+}

package/src/benchmark/invariants.js

@@ -10,6 +10,8 @@
 
 import { join } from "node:path";
 
+import { buildHookEnv } from "./hook-env.js";
+
 /**
  * @typedef {object} InvariantsResult
  * @property {"pass" | "fail"} verdict
@@ -20,7 +22,7 @@ import { join } from "node:path";
 /**
  * Run the task's invariants script.
  * @param {import("./task-family.js").Task} task
- * @param {{cwd: string, port: number, runDir: string}} ctx
+ * @param {{cwd: string, port: number, runDir: string, familyDir?: string|null}} ctx
  * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {Promise<InvariantsResult>}
  */
@@ -44,9 +46,14 @@ export async function runInvariants(task, ctx, runtime) {
   try {
     child = runtime.subprocess.spawn(script, [], {
       env: {
-        ...runtime.proc.env,
-        WORKDIR: ctx.cwd,
-        PORT: String(ctx.port),
+        ...buildHookEnv(runtime.proc.env, {
+          cwd: ctx.cwd,
+          port: ctx.port,
+          taskId: task.id,
+          taskDir: task.paths.taskDir,
+          hooksDir: task.paths.hooks,
+          familyDir: ctx.familyDir,
+        }),
         RESULTS_FD: "3",
       },
       stdio: ["inherit", "pipe", "pipe", fd3File],

package/src/benchmark/runner.js

@@ -201,6 +201,7 @@ export class BenchmarkRunner {
           cwd: workdir.cwd,
           port: workdir.port,
           runDir: workdir.runDir,
+          familyDir: family.rootPath,
         },
         this.runtime,
       );

package/src/benchmark/workdir.js

@@ -17,6 +17,7 @@ import { connect } from "node:net";
 import { join } from "node:path";
 
 import { loadEnv } from "./env-loader.js";
+import { buildHookEnv } from "./hook-env.js";
 
 const DEFAULT_TERM_GRACE_MS = 5_000;
 
@@ -73,6 +74,24 @@ export class WorkdirManager {
     const cwd = join(runDir, "cwd");
     await fs.mkdir(cwd, { recursive: true });
 
+    // Family-level shared fixtures: convention-over-configuration, copied if
+    // present. They form the shared base; the per-task workdir/specs below
+    // overlay on top (fs.cp defaults to force:true, so a per-task file wins).
+    if (this.familyRootPath) {
+      await fs
+        .cp(join(this.familyRootPath, "workdir"), cwd, { recursive: true })
+        .catch((e) => {
+          if (e.code !== "ENOENT") throw e;
+        });
+      await fs
+        .cp(join(this.familyRootPath, "specs"), join(cwd, "specs"), {
+          recursive: true,
+        })
+        .catch((e) => {
+          if (e.code !== "ENOENT") throw e;
+        });
+    }
+
     await fs.cp(task.paths.workdir, cwd, { recursive: true }).catch((e) => {
       if (e.code !== "ENOENT") throw e;
     });
@@ -107,7 +126,12 @@ export class WorkdirManager {
     const judgeTracePath = join(runDir, "judge.ndjson");
 
     const preflight = task.paths.preflight
-      ? await runPreflight(this.runtime, task.paths.preflight, cwd, port)
+      ? await runPreflight(this.runtime, task.paths.preflight, cwd, port, {
+          taskId: task.id,
+          taskDir: task.paths.taskDir,
+          hooksDir: task.paths.hooks,
+          familyDir: this.familyRootPath,
+        })
       : { pgid: 0 };
 
     return {
@@ -163,12 +187,13 @@ export class WorkdirManager {
  * @param {string} script
  * @param {string} cwd - Agent CWD passed via $WORKDIR.
  * @param {number} port - Free TCP port passed via $PORT.
+ * @param {{taskId: string, taskDir: string, hooksDir: string, familyDir: string|null}} vars - Extra hook env vars.
  * @returns {Promise<{pgid: number, error?: {phase: string, message: string, exitCode: number}}>}
  */
-async function runPreflight(runtime, script, cwd, port) {
+async function runPreflight(runtime, script, cwd, port, vars) {
   const child = runtime.subprocess.spawn(script, [], {
     cwd,
-    env: { ...runtime.proc.env, WORKDIR: cwd, PORT: String(port) },
+    env: buildHookEnv(runtime.proc.env, { cwd, port, ...vars }),
     detached: true,
     stdio: ["ignore", "pipe", "pipe"],
   });

package/src/supervisor.js

@@ -30,8 +30,7 @@ import { OrchestrationLoop } from "./orchestration-loop.js";
 /** System prompt for the supervisor lead. L0 mechanics only per COALIGNED. */
 export const SUPERVISOR_SYSTEM_PROMPT =
   "You supervise one agent.\n" +
-  "You have no tools to perform work yourself.\n" +
-  "Use `Ask` to delegate work to the agent.\n" +
+  "Use `Ask` to delegate the agent's task to the agent.\n" +
   "`Ask` is async and returns {askIds:[N]} immediately.\n" +
   "The reply arrives on your next turn as `[answer#N] agent: <text>` in your inbox.\n" +
   "End your turn while Asks are pending. The system resumes you when an answer arrives.\n" +
@@ -196,7 +195,6 @@ export function createSupervisor({
     "Task",
     "TaskOutput",
     "TaskStop",
-    "Bash",
     "Write",
     "Edit",
   ];
@@ -210,7 +208,7 @@ export function createSupervisor({
     output: devNull,
     model: supervisorModel ?? model,
     maxTurns: perRunBudget,
-    allowedTools: supervisorAllowedTools ?? ["Read", "Glob", "Grep"],
+    allowedTools: supervisorAllowedTools ?? ["Read", "Glob", "Grep", "Bash"],
     disallowedTools,
     onLine: (line) => supervisor.emitLine("supervisor", line),
     settingSources: ["project"],