@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/scripts/eval/replay/run.mjs

@@ -0,0 +1,404 @@
+#!/usr/bin/env node
+/**
+ * scripts/eval/replay/run.mjs — replay task fixtures on engine `claude` and
+ * engine `cohort`, then grade and report (design §8.4.2).
+ *
+ * Each task × engine runs in a fresh temp sandbox:
+ *   agent/          the agent root: with --agent-root, the seat's instruction and
+ *                   behaviour surface (lib/sandbox.mjs: CLAUDE.md, .claude/,
+ *                   config/agent.json, agents/, skills/, scripts/hooks/,
+ *                   policies/ — never .env, state/, logs/ or credentials); the
+ *                   fixture's files over it; and a .mcp.json holding ONLY the
+ *                   record/replay stub (mcp-replay-stub.mjs) — the seat's other
+ *                   servers are dropped and named in meta.json
+ *   home/           HOME and CLAUDE_CONFIG_DIR — never the real ones — with the
+ *                   seat's global settings as `maestro setup` writes them
+ *                   (the pre-send-audit.sh PreToolUse hook on outbound tools)
+ *   runtime/        MAESTRO_RUNTIME_DIR: the send gate's counter and send log
+ *   sessions/       COHORT_ENGINE_SESSIONS_DIR
+ * {{AGENT_ROOT}} and {{HOME}} in a fixture become the sandbox's paths. The
+ * spawn is the runtime adapter's (`buildSpawn`, the task's lane, the daemon's
+ * strict MCP flags), so what runs is what the daemon would run — except the
+ * permission posture, which is scoped rather than a seat's bypass: dontAsk,
+ * read tools and the org server allowed, Bash / writes / web / spawned work
+ * denied (lib/sandbox.mjs replayPermissions; a fixture's `sandbox.permissions:
+ * "bypass"` needs --allow-bypass). Tool calls are answered from the fixture;
+ * outbound sends are recorded by the stub and never executed. The child runs
+ * in its own process group: a run past its timeout gets SIGTERM then SIGKILL,
+ * and whatever it left behind is killed when it exits.
+ *
+ * Output (private; under state/): <out>/meta.json, <out>/tasks/<id>.json,
+ * <out>/records/<engine>/<id>.json, then graded.json, report.json and report.md.
+ *
+ * CLI:
+ *   node scripts/eval/replay/run.mjs [--tasks <dir|file>]... [--examples] [--engines claude,cohort]
+ *     [--agent-root <seat>] [--out <dir>] [--claude-bin <path>] [--base-url <url>] [--wire openai|anthropic]
+ *     [--concurrency <n>] [--judge] [--judge-tier <tier>] [--no-grade] [--keep-sandbox] [--require-pass]
+ *     [--allow-bypass] [--pass-env NAME]...
+ *   The gateway token comes from COHORT_LLM_TOKEN (never a flag). Engine cohort
+ *   needs a deployed gateway and a non-production token (human gate G8).
+ *
+ * @module scripts/eval/replay/run
+ */
+
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { buildSpawn } from "../../../lib/runtime/adapter.mjs";
+import { resolveClaudeBin } from "../../../lib/claude-bin.mjs";
+import { mergeSessionSettings } from "../../../lib/collective/global-config.mjs";
+import { validateTask } from "./lib/fixture.mjs";
+import { copySeat, fillPlaceholders, replayPermissions, seatMcpServerNames } from "./lib/sandbox.mjs";
+import { REPO, STATE_PATHS, ENGINES, PRIVACY_WARNING } from "./lib/config.mjs";
+import { gradeRunDir } from "./grade.mjs";
+import { writeReport } from "./report.mjs";
+
+export const STUB_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "mcp-replay-stub.mjs");
+export const EXAMPLES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "examples");
+const OUTPUT_CAP = 2_000_000;
+const KILL_GRACE_MS = 5_000;
+
+/**
+ * Load and validate task fixtures from files or directories of *.json.
+ * @param {string[]} inputs
+ * @returns {{tasks:any[], errors:string[]}}
+ */
+export function loadTasks(inputs) {
+  const tasks = [];
+  const errors = [];
+  const seen = new Set();
+  for (const input of inputs) {
+    const files = existsSync(input) && statSync(input).isDirectory() ? readdirSync(input).filter((f) => f.endsWith(".json")).sort().map((f) => path.join(input, f)) : [input];
+    for (const file of files) {
+      let raw;
+      try {
+        raw = JSON.parse(readFileSync(file, "utf8"));
+      } catch (e) {
+        errors.push(`${file}: ${e instanceof Error ? e.message : e}`);
+        continue;
+      }
+      const v = validateTask(raw);
+      if (!v.ok) errors.push(`${file}: ${v.errors.join("; ")}`);
+      else if (seen.has(v.task.id)) errors.push(`${file}: duplicate task id ${v.task.id}`);
+      else {
+        seen.add(v.task.id);
+        tasks.push(v.task);
+      }
+    }
+  }
+  return { tasks, errors };
+}
+
+/** The first balanced-looking JSON object in a CLI's stdout (a banner may precede it). */
+export function parseResultObject(/** @type string */ stdout) {
+  const s = String(stdout ?? "").trim();
+  const start = s.indexOf("{");
+  const end = s.lastIndexOf("}");
+  if (start < 0 || end <= start) return null;
+  try {
+    return JSON.parse(s.slice(start, end + 1));
+  } catch {
+    return null;
+  }
+}
+
+/** @param {string} file */
+function readJsonl(file) {
+  if (!existsSync(file)) return [];
+  return readFileSync(file, "utf8").split("\n").filter((l) => l.trim()).flatMap((l) => {
+    try {
+      return [JSON.parse(l)];
+    } catch {
+      return [];
+    }
+  });
+}
+
+/** @param {string} dir @param {string} name */
+function findFile(dir, name) {
+  if (!existsSync(dir)) return null;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, entry.name);
+    if (entry.isFile() && entry.name === name) return p;
+    if (entry.isDirectory()) {
+      const hit = findFile(p, name);
+      if (hit) return hit;
+    }
+  }
+  return null;
+}
+
+/**
+ * Build the sandbox for one task × engine. Returns its paths and env.
+ * @param {any} task @param {string} engine @param {Record<string,string>} [extraEnv]
+ */
+export function createSandbox(task, engine, extraEnv = {}, o = {}) {
+  const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), `replay-${engine}-`)));
+  const p = { root, agent: path.join(root, "agent"), home: path.join(root, "home"), runtime: path.join(root, "runtime"), sessions: path.join(root, "sessions"), tmp: path.join(root, "tmp") };
+  for (const dir of Object.values(p)) mkdirSync(dir, { recursive: true });
+  const claudeDir = path.join(p.home, ".claude");
+  mkdirSync(claudeDir, { recursive: true });
+  // The seat's instruction and behaviour surface first, the fixture's files over it.
+  const seat = o.seatRoot ? { ...copySeat(o.seatRoot, p.agent), droppedMcpServers: seatMcpServerNames(o.seatRoot).filter((n) => n !== task.mcp.server) } : null;
+  task = fillPlaceholders(task, { agentRoot: p.agent, home: p.home });
+  for (const [rel, content] of Object.entries(task.agentRoot.files)) {
+    const target = path.resolve(p.agent, rel);
+    if (!target.startsWith(p.agent + path.sep)) throw new Error(`task ${task.id}: agentRoot file ${rel} escapes the agent root`);
+    mkdirSync(path.dirname(target), { recursive: true });
+    writeFileSync(target, String(content));
+  }
+  const taskFile = path.join(root, "task.json");
+  const mcpLog = path.join(root, "mcp-calls.jsonl");
+  writeFileSync(taskFile, JSON.stringify(task));
+  writeFileSync(path.join(p.agent, ".mcp.json"), JSON.stringify({ mcpServers: { [task.mcp.server]: { command: process.execPath, args: [STUB_PATH], env: { REPLAY_TASK: taskFile, REPLAY_LOG: mcpLog } } } }, null, 2));
+  // The seat's global settings as `maestro setup` merges them, pointing at this SDK's hooks.
+  writeFileSync(path.join(claudeDir, "settings.json"), JSON.stringify(mergeSessionSettings({}, { agentRoot: REPO }).settings, null, 2));
+  const env = {
+    PATH: process.env.PATH ?? "/usr/bin:/bin",
+    HOME: p.home,
+    CLAUDE_CONFIG_DIR: claudeDir,
+    TMPDIR: p.tmp,
+    AGENT_ROOT: p.agent,
+    AGENT_DIR: p.agent,
+    MAESTRO_RUNTIME_DIR: p.runtime,
+    MAESTRO_DAEMON: "1",
+    COHORT_ENGINE_SESSIONS_DIR: p.sessions,
+    ...extraEnv,
+  };
+  return { ...p, claudeDir, taskFile, mcpLog, env, task, seat };
+}
+
+/** Allowed send-gate log entries of a sandbox's runtime dir. @param {string} runtimeDir */
+function readGateLog(runtimeDir) {
+  const auditDir = path.join(runtimeDir, "audit");
+  if (!existsSync(auditDir)) return [];
+  return readdirSync(auditDir).filter((f) => f.endsWith("-sends.jsonl")).sort().flatMap((f) => readJsonl(path.join(auditDir, f))).filter((r) => r.status === "allowed").map((r) => ({ timestamp: r.timestamp ?? null, channel: r.channel ?? null, recipient: r.recipient ?? "" }));
+}
+
+/**
+ * Run one task on one engine.
+ * @param {any} task a validated task
+ * @param {'claude'|'cohort'} engine
+ * @param {{claudeBin?:string, gateway?:{baseUrl:string, token:string, wire?:string}|null, permissions?:string[], allowBypass?:boolean, seatRoot?:string|null, env?:Record<string,string>, keepSandbox?:boolean}} o
+ *   permissions overrides the posture (tests only); seatRoot copies a seat's surface into the sandbox
+ */
+export async function runTask(task, engine, o = {}) {
+  const box = createSandbox(task, engine, { ...(o.env ?? {}), ...(engine === "cohort" && o.gateway?.wire ? { COHORT_LLM_WIRE: o.gateway.wire } : {}) }, { seatRoot: o.seatRoot ?? null });
+  const perms = o.permissions ? { posture: "override", note: null, args: o.permissions } : replayPermissions(box.task, { allowBypass: o.allowBypass === true });
+  const sessionId = randomUUID();
+  const started = Date.now();
+  /** @type {any} */
+  const record = { taskId: task.id, engine, sessionId, exitCode: null, signal: null, timedOut: false, durationMs: 0, result: null, stdoutTail: "", stderrTail: "", mcpCalls: [], gateSends: 0, gateLog: [], transcript: null, error: null, sandboxRoot: box.root, posture: perms.posture, postureNote: perms.note, permissionArgs: perms.args, seat: box.seat ? { copied: box.seat.copied, droppedMcpServers: box.seat.droppedMcpServers } : null };
+  try {
+    const spec = buildSpawn(
+      {
+        lane: task.lane,
+        engine,
+        model: task.model,
+        prompt: box.task.prompt,
+        sessionId,
+        agentRoot: box.agent,
+        permissions: perms.args,
+        mcp: { agentRoot: box.agent },
+        knobs: { maxTurns: task.maxTurns },
+        env: box.env,
+        ...(engine === "cohort" ? { cohortToken: o.gateway?.token, cohortBaseUrl: o.gateway?.baseUrl } : {}),
+      },
+      { daemonArgs: (/** @type string */ root) => ["--strict-mcp-config", "--mcp-config", path.join(root, ".mcp.json")], resolveBin: () => o.claudeBin ?? resolveClaudeBin() },
+    );
+    if (!spec.ok) {
+      record.error = `${spec.error.code}: ${spec.error.message}`;
+      return record;
+    }
+    const outcome = await new Promise((resolve) => {
+      // Its own process group, so a timeout or the end of the run reaches the
+      // shells, monitors and MCP servers it started, not only the direct child.
+      const child = spawn(spec.bin, spec.argv, { cwd: box.agent, env: spec.env, stdio: [spec.stdin === null ? "ignore" : "pipe", "pipe", "pipe"], detached: true });
+      const killGroup = (/** @type NodeJS.Signals */ sig) => {
+        try {
+          if (child.pid) process.kill(-child.pid, sig);
+        } catch {
+          try {
+            child.kill(sig);
+          } catch {
+            /* already gone */
+          }
+        }
+      };
+      let stdout = "";
+      let stderr = "";
+      let timedOut = false;
+      child.stdout?.on("data", (c) => (stdout = (stdout + c).slice(-OUTPUT_CAP)));
+      child.stderr?.on("data", (c) => (stderr = (stderr + c).slice(-OUTPUT_CAP)));
+      child.stdin?.on("error", () => {});
+      if (spec.stdin !== null) child.stdin?.end(spec.stdin);
+      /** @type {NodeJS.Timeout|undefined} */
+      let hardKill;
+      const timer = setTimeout(() => {
+        timedOut = true;
+        killGroup("SIGTERM");
+        hardKill = setTimeout(() => killGroup("SIGKILL"), KILL_GRACE_MS);
+      }, task.timeoutMs);
+      // The leader exited: nothing it left behind may outlive the run (or hold its pipes open).
+      child.on("exit", () => killGroup("SIGKILL"));
+      child.on("error", (e) => {
+        clearTimeout(timer);
+        clearTimeout(hardKill);
+        resolve({ code: null, signal: null, stdout, stderr: `${stderr}\n${e.message}`, timedOut, spawnError: e.message });
+      });
+      child.on("close", (code, signal) => {
+        clearTimeout(timer);
+        clearTimeout(hardKill);
+        killGroup("SIGKILL");
+        resolve({ code, signal, stdout, stderr, timedOut });
+      });
+    });
+    const out = /** @type any */ (outcome);
+    record.exitCode = out.code;
+    record.signal = out.signal;
+    record.timedOut = out.timedOut;
+    record.error = out.spawnError ?? null;
+    record.result = parseResultObject(out.stdout);
+    record.stdoutTail = out.stdout.slice(-4000);
+    record.stderrTail = out.stderr.slice(-4000);
+    record.mcpCalls = readJsonl(box.mcpLog);
+    record.gateLog = readGateLog(box.runtime);
+    record.gateSends = record.gateLog.length;
+    const sid = typeof record.result?.session_id === "string" ? record.result.session_id : sessionId;
+    const transcriptFile = engine === "cohort" ? findFile(box.sessions, `${sid}.jsonl`) : findFile(path.join(box.claudeDir, "projects"), `${sid}.jsonl`);
+    record.transcript = transcriptFile ? readFileSync(transcriptFile, "utf8") : null;
+    return record;
+  } catch (e) {
+    record.error = e instanceof Error ? e.message : String(e);
+    return record;
+  } finally {
+    record.durationMs = Date.now() - started;
+    if (!o.keepSandbox) rmSync(box.root, { recursive: true, force: true });
+    else record.sandbox = box.root;
+  }
+}
+
+/**
+ * Run every task on every engine and write the records.
+ * @param {{tasks:any[], engines:string[], outDir:string, concurrency?:number, onRecord?:(r:any)=>void} & Parameters<typeof runTask>[2]} o
+ */
+export async function runReplay(o) {
+  mkdirSync(path.join(o.outDir, "tasks"), { recursive: true });
+  for (const t of o.tasks) writeFileSync(path.join(o.outDir, "tasks", `${t.id}.json`), `${JSON.stringify(t, null, 2)}\n`);
+  const jobs = o.tasks.flatMap((t) => o.engines.map((engine) => ({ t, engine })));
+  const records = [];
+  await Promise.all(
+    Array.from({ length: Math.max(1, Math.min(o.concurrency ?? 2, jobs.length)) }, async () => {
+      for (let job = jobs.shift(); job; job = jobs.shift()) {
+        const r = await runTask(job.t, /** @type any */ (job.engine), o);
+        const dir = path.join(o.outDir, "records", job.engine);
+        mkdirSync(dir, { recursive: true });
+        writeFileSync(path.join(dir, `${job.t.id}.json`), `${JSON.stringify(r, null, 2)}\n`);
+        records.push(r);
+        o.onRecord?.(r);
+      }
+    }),
+  );
+  return records;
+}
+
+/**
+ * Refuse an output directory inside this repository unless it is under state/ (gitignored).
+ * @param {string} dir absolute
+ */
+export function outputDirProblem(dir, repo = REPO) {
+  const rel = path.relative(repo, dir);
+  if (rel.startsWith("..") || path.isAbsolute(rel)) return null;
+  return rel === "state" || rel.startsWith(`state${path.sep}`) ? null : `refusing to write eval output to ${dir}: inside the repository only state/ is gitignored`;
+}
+
+/**
+ * @param {string[]} argv
+ * @param {{env?:NodeJS.ProcessEnv, stdout?:{write:(s:string)=>void}, stderr?:{write:(s:string)=>void}, now?:()=>Date, fetchImpl?:typeof fetch}} [deps]
+ */
+export async function main(argv, deps = {}) {
+  const env = deps.env ?? process.env;
+  const out = deps.stdout ?? process.stdout;
+  const err = deps.stderr ?? process.stderr;
+  const values = (/** @type string */ f) => argv.flatMap((a, i) => (a === f && argv[i + 1] !== undefined ? [argv[i + 1]] : []));
+  const value = (/** @type string */ f) => values(f).at(-1);
+  err.write(`${PRIVACY_WARNING}\n`);
+
+  // CF-15: the synthetic examples are fixtures, kept in git and left out of the npm package.
+  if (argv.includes("--examples") && !existsSync(EXAMPLES_DIR)) {
+    err.write(`replay: --examples needs ${EXAMPLES_DIR}, which ships only in a git checkout of the SDK (not in the npm package); pass --tasks <dir> instead\n`);
+    return 2;
+  }
+  const inputs = [...values("--tasks"), ...(argv.includes("--examples") ? [EXAMPLES_DIR] : [])];
+  if (inputs.length === 0) inputs.push(path.join(REPO, STATE_PATHS.tasks));
+  const { tasks, errors } = loadTasks(inputs);
+  if (errors.length > 0) {
+    err.write(`replay: invalid tasks:\n${errors.map((e) => `  - ${e}`).join("\n")}\n`);
+    return 2;
+  }
+  if (tasks.length === 0) {
+    err.write(`replay: no tasks in ${inputs.join(", ")} (extract some with scripts/eval/replay/extract.mjs, or pass --examples)\n`);
+    return 2;
+  }
+  const engines = (value("--engines") ?? ENGINES.join(",")).split(",").map((s) => s.trim()).filter(Boolean);
+  const unknown = engines.filter((e) => !ENGINES.includes(/** @type any */ (e)));
+  if (unknown.length) {
+    err.write(`replay: unknown engine(s) ${unknown.join(", ")}\n`);
+    return 2;
+  }
+  const baseUrl = value("--base-url") ?? env.COHORT_LLM_BASE_URL ?? null;
+  const token = env.COHORT_LLM_TOKEN ?? null;
+  if (engines.includes("cohort") && (!baseUrl || !token)) {
+    err.write("replay: BLOCKED — engine cohort needs a deployed cohort-llm gateway (--base-url or COHORT_LLM_BASE_URL) and a non-production seat token in COHORT_LLM_TOKEN (human gate G8). Run --engines claude alone, or supply both.\n");
+    return 2;
+  }
+  const stamp = (deps.now ? deps.now() : new Date()).toISOString().replace(/[:.]/g, "-");
+  const outDir = path.resolve(value("--out") ?? path.join(REPO, STATE_PATHS.runs, stamp));
+  const problem = outputDirProblem(outDir);
+  if (problem) {
+    err.write(`replay: ${problem}\n`);
+    return 2;
+  }
+  const seatRoot = value("--agent-root") ? path.resolve(/** @type string */ (value("--agent-root"))) : null;
+  if (seatRoot && !(existsSync(seatRoot) && statSync(seatRoot).isDirectory())) {
+    err.write(`replay: --agent-root ${seatRoot} is not a directory\n`);
+    return 2;
+  }
+  const seatProblem = seatRoot ? outputDirProblem(outDir, seatRoot)?.replace("the repository", "the seat") : null;
+  if (seatProblem) {
+    err.write(`replay: ${seatProblem}\n`);
+    return 2;
+  }
+  const concurrency = Math.max(1, Number(value("--concurrency")) || 2);
+  const allowBypass = argv.includes("--allow-bypass");
+  const judgeRequested = argv.includes("--judge");
+  // The sandbox env is built from nothing; --pass-env NAME forwards one variable (never a credential by accident: named explicitly).
+  const passEnv = Object.fromEntries(values("--pass-env").flatMap((name) => (env[name] !== undefined ? [[name, String(env[name])]] : [])));
+  out.write(`replay: ${tasks.length} task(s) × ${engines.join(", ")} → ${outDir}\n`);
+  mkdirSync(outDir, { recursive: true });
+  const meta = { startedAt: (deps.now ? deps.now() : new Date()).toISOString(), engines, tasks: tasks.length, agentRoot: seatRoot, seatMcpServersDropped: seatRoot ? seatMcpServerNames(seatRoot).filter((n) => !tasks.some((t) => t.mcp.server === n)) : [], posture: allowBypass ? "scoped; bypass for fixtures that opt in" : "scoped", concurrency, judge: judgeRequested, passEnv: Object.keys(passEnv) };
+  writeFileSync(path.join(outDir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`);
+  await runReplay({
+    tasks,
+    engines,
+    outDir,
+    concurrency,
+    env: passEnv,
+    seatRoot,
+    allowBypass,
+    claudeBin: value("--claude-bin"),
+    gateway: baseUrl && token ? { baseUrl, token, wire: value("--wire") } : null,
+    keepSandbox: argv.includes("--keep-sandbox"),
+    onRecord: (r) => out.write(`  ${r.engine.padEnd(6)} ${r.taskId}: exit ${r.exitCode}${r.timedOut ? " (timed out)" : ""}${r.error ? ` — ${r.error}` : ""}\n`),
+  });
+  if (argv.includes("--no-grade")) return 0;
+  await gradeRunDir(outDir, { judge: judgeRequested, judgeTier: value("--judge-tier"), baseUrl, token: env.COHORT_EVAL_JUDGE_TOKEN ?? token, fetchImpl: deps.fetchImpl });
+  const { report, md } = writeReport(outDir);
+  out.write(md);
+  return argv.includes("--require-pass") && report.gate.verdict !== "pass" ? 1 : 0;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) process.exitCode = await main(process.argv.slice(2));