@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/scripts/eval/replay/lib/fixture.mjs

@@ -0,0 +1,188 @@
+/**
+ * scripts/eval/replay/lib/fixture.mjs — the replay task fixture (schema v1).
+ *
+ * {
+ *   schemaVersion: 1,
+ *   id: "ex-01-board-status",               file-name safe
+ *   synthetic: true,                          examples only; extracted tasks are false
+ *   source: { kind: "synthetic"|"dispatcher"|"cadence", sessionId?, recordedAt?, cadence? },
+ *   lane: "dispatcher"|"cadence",             which maestro lane ran it
+ *   prompt: "…",
+ *   model?: "sonnet",                         the lane's --model (the engine maps it to a tier)
+ *   maxTurns?: 20, timeoutMs?: 180000,
+ *   agentRoot?: { files: { "CLAUDE.md": "…" } },   written into the sandbox agent root, over a copied seat
+ *   sandbox?: { permissions?: "scoped"|"bypass", allowTools?: ["Bash(ls:*)"] },   see lib/sandbox.mjs
+ *   mcp: {
+ *     server: "cohort",
+ *     tools: ["board_mine", {name, description, inputSchema}],   names resolve against the org tool surface
+ *     replies: [{ tool: "board_mine", input?: {…partial matcher}, recordedInput?: {…the recorded call}, result: "text" | {text}, isError?: false }]
+ *   },
+ *   expect: {
+ *     toolCalls?: [{ tool: "mcp__cohort__board_mine", input?: {…partial}, min?: 1, max? }],
+ *     forbidden?: [{ tool: "mcp__cohort__email_send", input?: {…partial} }],
+ *     finalAnswer?: { contains?: [], notContains?: [], matches?: [], nonEmpty?: true },
+ *     rubric?: "what a good answer does", reference?: "a redacted recorded answer"
+ *   }
+ * }
+ *
+ * Pure except `resolveToolDefs`, which reads the org tool table (a module, no I/O).
+ *
+ * @module scripts/eval/replay/lib/fixture
+ */
+
+import { SEND_GATE_TOOLS, SEND_GATE_MATCHER } from "../../../../lib/engine/hooks.mjs";
+import { orgToolDef } from "../../../../lib/org/tool-surface.mjs";
+import { DEFAULT_MAX_TURNS, DEFAULT_TASK_TIMEOUT_MS } from "./config.mjs";
+
+export const FIXTURE_SCHEMA_VERSION = 1;
+export const OUTBOUND_TOOLS = SEND_GATE_TOOLS;
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
+
+/** Is this tool name (full `mcp__server__tool` or bare) an outbound send? */
+export function isOutboundTool(/** @type string */ name) {
+  return SEND_GATE_MATCHER.test(name) || OUTBOUND_TOOLS.includes(name);
+}
+
+/** @param {string} server @param {string} tool */
+export const fullToolName = (server, tool) => (tool.startsWith("mcp__") ? tool : `mcp__${server}__${tool}`);
+
+/**
+ * Is `partial` contained in `value`? Objects by key, arrays by position,
+ * strings exactly (or by a {"$contains": "…"} / {"$regex": "…"} matcher).
+ * @param {unknown} partial @param {unknown} value
+ */
+export function partialMatch(partial, value) {
+  if (partial === undefined) return true;
+  if (partial && typeof partial === "object" && !Array.isArray(partial)) {
+    const p = /** @type {Record<string, unknown>} */ (partial);
+    if (typeof p.$contains === "string") return typeof value === "string" && value.toLowerCase().includes(p.$contains.toLowerCase());
+    if (typeof p.$regex === "string") return typeof value === "string" && new RegExp(p.$regex, "i").test(value);
+    if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+    return Object.entries(p).every(([k, v]) => partialMatch(v, /** @type {Record<string, unknown>} */ (value)[k]));
+  }
+  if (Array.isArray(partial)) return Array.isArray(value) && partial.every((v, i) => partialMatch(v, value[i]));
+  return JSON.stringify(partial) === JSON.stringify(value);
+}
+
+/**
+ * @param {any} task
+ * @returns {{ok:true, task:any} | {ok:false, errors:string[]}}
+ */
+export function validateTask(task) {
+  const errors = [];
+  if (!task || typeof task !== "object") return { ok: false, errors: ["a task must be an object"] };
+  if (task.schemaVersion !== FIXTURE_SCHEMA_VERSION) errors.push(`schemaVersion must be ${FIXTURE_SCHEMA_VERSION}`);
+  if (typeof task.id !== "string" || !ID_RE.test(task.id)) errors.push("id must be a file-name-safe string");
+  if (task.lane !== "dispatcher" && task.lane !== "cadence") errors.push('lane must be "dispatcher" or "cadence"');
+  if (typeof task.prompt !== "string" || task.prompt.trim() === "") errors.push("prompt is required");
+  if (task.maxTurns !== undefined && !(Number.isInteger(task.maxTurns) && task.maxTurns > 0)) errors.push("maxTurns must be a positive integer");
+  if (task.timeoutMs !== undefined && !(Number.isInteger(task.timeoutMs) && task.timeoutMs > 0)) errors.push("timeoutMs must be a positive integer");
+  const mcp = task.mcp ?? {};
+  if (typeof mcp.server !== "string" || !/^[A-Za-z0-9_-]+$/.test(mcp.server)) errors.push("mcp.server must be a server name");
+  if (!Array.isArray(mcp.tools)) errors.push("mcp.tools must be an array");
+  if (mcp.replies !== undefined && !Array.isArray(mcp.replies)) errors.push("mcp.replies must be an array");
+  for (const [i, r] of (Array.isArray(mcp.replies) ? mcp.replies : []).entries()) {
+    if (typeof r?.tool !== "string") errors.push(`mcp.replies[${i}].tool is required`);
+    if (typeof r?.result !== "string" && typeof r?.result?.text !== "string") errors.push(`mcp.replies[${i}].result must be text or {text}`);
+  }
+  const sb = task.sandbox;
+  if (sb !== undefined) {
+    if (!sb || typeof sb !== "object") errors.push("sandbox must be an object");
+    else {
+      if (sb.permissions !== undefined && sb.permissions !== "scoped" && sb.permissions !== "bypass") errors.push('sandbox.permissions must be "scoped" or "bypass"');
+      if (sb.allowTools !== undefined && !(Array.isArray(sb.allowTools) && sb.allowTools.every((/** @type unknown */ r) => typeof r === "string"))) errors.push("sandbox.allowTools must be an array of rules");
+    }
+  }
+  const expect = task.expect ?? {};
+  for (const key of ["toolCalls", "forbidden"]) {
+    if (expect[key] !== undefined && !Array.isArray(expect[key])) errors.push(`expect.${key} must be an array`);
+    for (const [i, c] of (Array.isArray(expect[key]) ? expect[key] : []).entries()) if (typeof c?.tool !== "string") errors.push(`expect.${key}[${i}].tool is required`);
+  }
+  const fa = expect.finalAnswer;
+  if (fa !== undefined) {
+    for (const key of ["contains", "notContains", "matches"]) if (fa[key] !== undefined && !Array.isArray(fa[key])) errors.push(`expect.finalAnswer.${key} must be an array`);
+    for (const re of fa.matches ?? []) {
+      try {
+        new RegExp(re);
+      } catch {
+        errors.push(`expect.finalAnswer.matches: ${JSON.stringify(re)} is not a regular expression`);
+      }
+    }
+  }
+  if (errors.length > 0) return { ok: false, errors };
+  return {
+    ok: true,
+    task: {
+      ...task,
+      synthetic: task.synthetic === true,
+      model: task.model ?? "sonnet",
+      maxTurns: task.maxTurns ?? DEFAULT_MAX_TURNS,
+      timeoutMs: task.timeoutMs ?? DEFAULT_TASK_TIMEOUT_MS,
+      agentRoot: { files: { ...(task.agentRoot?.files ?? {}) } },
+      sandbox: { permissions: sb?.permissions ?? "scoped", allowTools: [...(sb?.allowTools ?? [])] },
+      mcp: { server: mcp.server, tools: mcp.tools, replies: mcp.replies ?? [] },
+      expect: { toolCalls: expect.toolCalls ?? [], forbidden: expect.forbidden ?? [], finalAnswer: expect.finalAnswer ?? {}, rubric: expect.rubric ?? null, reference: expect.reference ?? null },
+    },
+  };
+}
+
+/**
+ * The MCP tool definitions a task's server lists: named org tools take their
+ * real schema from lib/org/tool-surface.mjs; inline definitions pass through.
+ * @param {{mcp:{tools:Array<string|{name:string, description?:string, inputSchema?:object}>}}} task
+ * @returns {Array<{name:string, description:string, inputSchema:object, readOnly:boolean}>}
+ */
+export function resolveToolDefs(task) {
+  return task.mcp.tools.map((t) => {
+    if (typeof t === "object" && t) return { name: t.name, description: t.description ?? "", inputSchema: t.inputSchema ?? { type: "object" }, readOnly: false };
+    const def = /** @type any */ (orgToolDef(t));
+    if (!def) return { name: t, description: "", inputSchema: { type: "object" }, readOnly: false };
+    return { name: t, description: def.description ?? "", inputSchema: def.input_schema ?? def.inputSchema ?? { type: "object" }, readOnly: def.access === "read" };
+  });
+}
+
+/**
+ * Argument keys that pick WHICH record a call reads (ids, channels, threads,
+ * deals, invoices …), as opposed to how (paging, filters, free text).
+ */
+export const IDENTIFYING_KEY_RE = /^(?:id|ids|ts|thread_ts|number|slug|channel|thread|deal|invoice|bill|person|org|file|folder|doc|mailbox|message|task|item|event|decision|board|contact)$|[a-z0-9]Ids?$|_ids?$/;
+
+/**
+ * The identifying arguments of a call's input (top-level keys only). Pure.
+ * @param {unknown} input
+ * @returns {Record<string, unknown>|undefined} undefined when the input is not an object
+ */
+export function identifyingArgs(input) {
+  if (!input || typeof input !== "object" || Array.isArray(input)) return undefined;
+  return Object.fromEntries(Object.entries(input).filter(([k]) => IDENTIFYING_KEY_RE.test(k)));
+}
+
+/**
+ * Pick the reply for a call. A reply matches EXACTLY when its `input` matcher
+ * is contained in the call's arguments (and so is its `recordedInput`, when an
+ * extracted reply keeps one), and APPROXIMATELY when only the identifying
+ * arguments agree — another engine asked for the same record with different
+ * paging, filters or wording. Preference: unused exact, used exact (a repeated
+ * read), unused approximate, used approximate; among used replies the last
+ * wins. A call whose identifying arguments differ gets nothing. Pure over `used`.
+ * @param {Array<{tool:string, input?:any, recordedInput?:any, result:any, isError?:boolean}>} replies
+ * @param {string} tool bare tool name
+ * @param {any} args
+ * @param {Set<number>} used
+ * @returns {{index:number, match:'exact'|'approximate'}|null}
+ */
+export function pickReply(replies, tool, args, used) {
+  /** @type {{index:number, match:'exact'|'approximate', rank:number}|null} */
+  let best = null;
+  for (const [i, r] of replies.entries()) {
+    if (r.tool !== tool) continue;
+    /** @type {'exact'|'approximate'} */
+    let match;
+    if (partialMatch(r.input, args)) match = r.recordedInput === undefined || partialMatch(r.recordedInput, args) ? "exact" : "approximate";
+    else if (partialMatch(identifyingArgs(r.input), args)) match = "approximate";
+    else continue;
+    const rank = (match === "exact" ? 0 : 2) + (used.has(i) ? 1 : 0);
+    if (!best || rank < best.rank || (rank === best.rank && used.has(i))) best = { index: i, match, rank };
+  }
+  return best && { index: best.index, match: best.match };
+}

package/scripts/eval/replay/lib/judge.mjs

@@ -0,0 +1,72 @@
+/**
+ * scripts/eval/replay/lib/judge.mjs — the optional rubric judge.
+ *
+ * One request per graded answer to the cohort-llm gateway, on the pinned judge
+ * tier (config.mjs JUDGE), through the engine's own OpenAI wire client — the
+ * same contract, auth and cost headers as every engine request. Skipped (not
+ * failed) when the task has no rubric or reference, or when no gateway URL or
+ * token is configured. The prompt below is Cohort's own.
+ *
+ * @module scripts/eval/replay/lib/judge
+ */
+
+import { createModelCaller } from "../../../../lib/engine/wire/index.mjs";
+import { blockText } from "./transcript.mjs";
+import { JUDGE } from "./config.mjs";
+
+export const JUDGE_SYSTEM = [
+  "You grade whether a colleague's final answer accomplished a work task.",
+  "You are given the task, a rubric, sometimes a reference answer from an earlier run, and the candidate answer.",
+  "Judge substance, not wording or length: the candidate passes when it does what the rubric asks and claims nothing the task contradicts.",
+  "A reference answer shows one acceptable outcome, not the only one.",
+  'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.',
+].join("\n");
+
+/**
+ * @param {any} task @param {string} candidate
+ * @returns {{system:string, messages:Array<{role:'user', content:string}>}}
+ */
+export function buildJudgeRequest(task, candidate) {
+  const parts = [`Task:\n${task.prompt}`];
+  if (task.expect?.rubric) parts.push(`Rubric:\n${task.expect.rubric}`);
+  if (task.expect?.reference) parts.push(`Reference answer:\n${task.expect.reference}`);
+  parts.push(`Candidate answer:\n${candidate || "(empty)"}`);
+  return { system: JUDGE_SYSTEM, messages: [{ role: "user", content: parts.join("\n\n") }] };
+}
+
+/**
+ * @param {string} text
+ * @returns {{verdict:'pass'|'fail'|'error', score:number|null, reason:string}}
+ */
+export function parseJudgeVerdict(text) {
+  const s = String(text ?? "");
+  const start = s.indexOf("{");
+  const end = s.lastIndexOf("}");
+  if (start < 0 || end <= start) return { verdict: "error", score: null, reason: "the judge did not answer with JSON" };
+  try {
+    const o = JSON.parse(s.slice(start, end + 1));
+    if (typeof o.pass !== "boolean") return { verdict: "error", score: null, reason: "the judge's JSON has no boolean pass" };
+    const score = typeof o.score === "number" && Number.isFinite(o.score) ? Math.min(1, Math.max(0, o.score)) : null;
+    return { verdict: o.pass ? "pass" : "fail", score, reason: typeof o.reason === "string" ? o.reason.slice(0, 500) : "" };
+  } catch {
+    return { verdict: "error", score: null, reason: "the judge's JSON did not parse" };
+  }
+}
+
+/**
+ * @param {any} task
+ * @param {string} candidate
+ * @param {{baseUrl:string|null, token:string|null, tier?:string, fetchImpl?:typeof fetch, required?:boolean}} o
+ *   required: the run asked for the judge, so a missing gateway is an error, not a skip
+ * @returns {Promise<{verdict:'pass'|'fail'|'skipped'|'error', score:number|null, reason:string, tier:string}>}
+ */
+export async function judgeTask(task, candidate, o) {
+  const tier = o.tier ?? JUDGE.tier;
+  if (!task.expect?.rubric && !task.expect?.reference) return { verdict: "skipped", score: null, reason: "the task has no rubric or reference", tier };
+  if (!o.baseUrl || !o.token) return { verdict: o.required ? "error" : "skipped", score: null, reason: "no gateway URL or token for the judge", tier };
+  const call = createModelCaller({ wire: JUDGE.wire, baseUrl: o.baseUrl, token: o.token, model: tier, maxTokens: JUDGE.maxTokens, headers: { "x-cohort-surface": "eval-judge" }, fetchImpl: o.fetchImpl });
+  const req = buildJudgeRequest(task, candidate);
+  const r = await call({ system: req.system, messages: req.messages, tools: [] });
+  if (!r.ok) return { verdict: "error", score: null, reason: `judge request failed: ${r.error?.code ?? "unknown"}`, tier };
+  return { ...parseJudgeVerdict(blockText(r.value.message.content)), tier };
+}

package/scripts/eval/replay/lib/redact.mjs

@@ -0,0 +1,136 @@
+/**
+ * scripts/eval/replay/lib/redact.mjs — scrub a recorded session before it
+ * becomes a replay fixture. Pure.
+ *
+ * Simple patterns, applied in order: credentials first (so a token inside an
+ * email-looking string goes), then email addresses, phone numbers and chat
+ * ids, users' home directories in paths, then names — the string values of
+ * person-shaped JSON keys (name, displayName, author, from, to …, both in
+ * objects and in JSON text), the caller's list, the name parts of every email
+ * address seen, and a capitalised name after a greeting or sign-off. Known
+ * gaps: a name in free prose that is none of those, and JSON that is itself
+ * escaped inside a string (\"name\"). Redaction is
+ * deliberately greedy; a fixture that reads a little odd is fine, a leaked
+ * address is not. It is not a guarantee: fixtures stay private regardless.
+ *
+ * @module scripts/eval/replay/lib/redact
+ */
+
+/** @type {ReadonlyArray<{kind:string, re:RegExp, to:string}>} */
+export const SECRET_PATTERNS = Object.freeze([
+  { kind: "anthropic-key", re: /sk-ant-[A-Za-z0-9_-]{8,}/g, to: "[secret]" },
+  { kind: "seat-token", re: /\b(?:cst|nlk)_[A-Za-z0-9_-]{6,}/g, to: "[secret]" },
+  { kind: "slack-token", re: /\bxox[abposr]-[A-Za-z0-9-]{8,}/g, to: "[secret]" },
+  { kind: "github-token", re: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})/g, to: "[secret]" },
+  { kind: "aws-key", re: /\bAKIA[0-9A-Z]{16}\b/g, to: "[secret]" },
+  { kind: "jwt", re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, to: "[secret]" },
+  { kind: "bearer", re: /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, to: "Bearer [secret]" },
+  { kind: "api-key", re: /\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}/g, to: "[secret]" },
+  { kind: "long-hex", re: /\b[a-f0-9]{32,}\b/gi, to: "[secret]" },
+]);
+
+/** @type {ReadonlyArray<{kind:string, re:RegExp, to:string}>} */
+export const CONTACT_PATTERNS = Object.freeze([
+  { kind: "email", re: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, to: "[email]" },
+  { kind: "phone", re: /(?<![\w+])\+\d[\d\s().-]{7,}\d/g, to: "[phone]" },
+  { kind: "phone", re: /\(\d{3}\)\s?\d{3}[\s.-]\d{4}\b/g, to: "[phone]" },
+  { kind: "chat-id", re: /\b[CUDGW](?=[A-Z0-9]*\d)[A-Z0-9]{8,11}\b/g, to: "[chat-id]" },
+]);
+
+/** A user's directory in a filesystem path: /Users/<x>/ and /home/<x>/. */
+export const HOME_PATH_RE = /(\/(?:Users|home)\/)(?!\[user\])[^/\s"'`]+/g;
+
+/** JSON keys whose string values are people: name, displayName, author, from, to … */
+export const NAME_KEY_RE = /^(?:[A-Za-z0-9]+_)?(?:name|displayName|fullName|firstName|lastName|givenName|familyName|userName|username|author|sender|from|to|cc|bcc|assignee|owner|realName)$/i;
+const NAME_KEY_IN_TEXT = /("((?:[A-Za-z0-9]+_)?(?:name|displayName|fullName|firstName|lastName|givenName|familyName|userName|username|author|sender|from|to|cc|bcc|assignee|owner|realName))")(\s*:\s*)"(?:[^"\\\n]|\\.)*"/gi;
+
+const GREETING =/\b(Hi|Hello|Hey|Dear|Thanks|Thank you|Cheers|Regards|Best|Morning|Afternoon|Evening)(,?[ \t]+)([A-Z][a-z'’-]+(?:[ \t][A-Z][a-z'’-]+)?)/g;
+const MENTION = /(^|[\s(])@[A-Za-z0-9._-]{2,}/g;
+
+/** @param {string} s */
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Name parts of the email addresses in a text ("jane.doe@x" → jane, doe).
+ * @param {string} text
+ */
+export function namesFromEmails(text) {
+  const out = new Set();
+  for (const m of String(text).matchAll(/([A-Za-z0-9._%+-]+)@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g)) {
+    for (const part of m[1].split(/[._%+-]+/)) if (/^[A-Za-z]{3,}$/.test(part)) out.add(part);
+  }
+  return [...out];
+}
+
+/**
+ * @param {string} text
+ * @param {{names?:string[]}} [o]
+ * @returns {{text:string, counts:Record<string, number>}}
+ */
+export function redactText(text, o = {}) {
+  /** @type {Record<string, number>} */
+  const counts = {};
+  const bump = (/** @type string */ kind) => (counts[kind] = (counts[kind] ?? 0) + 1);
+  let s = String(text);
+  const names = new Set([...(o.names ?? []), ...namesFromEmails(s)].map((n) => String(n).trim()).filter((n) => n.length >= 2));
+  for (const p of [...SECRET_PATTERNS, ...CONTACT_PATTERNS]) {
+    s = s.replace(p.re, () => {
+      bump(p.kind);
+      return p.to;
+    });
+  }
+  s = s.replace(HOME_PATH_RE, (_m, lead) => {
+    bump("home-path");
+    return `${lead}[user]`;
+  });
+  s = s.replace(NAME_KEY_IN_TEXT, (m, key, _bare, sep) => {
+    if (m.endsWith('"[name]"')) return m;
+    bump("name");
+    return `${key}${sep}"[name]"`;
+  });
+  const sorted = [...names].sort((a, b) => b.length - a.length);
+  for (const name of sorted) {
+    const re = new RegExp(`(?<![A-Za-z0-9])${escapeRe(name)}(?![A-Za-z0-9])`, "gi");
+    s = s.replace(re, () => {
+      bump("name");
+      return "[name]";
+    });
+  }
+  s = s.replace(GREETING, (_m, greet, sep) => {
+    bump("name");
+    return `${greet}${sep}[name]`;
+  });
+  s = s.replace(MENTION, (_m, lead) => {
+    bump("handle");
+    return `${lead}@[handle]`;
+  });
+  return { text: s, counts };
+}
+
+/**
+ * Redact every string inside a JSON value (keys are kept).
+ * @param {unknown} value
+ * @param {{names?:string[]}} [o]
+ * @returns {{value:any, counts:Record<string, number>}}
+ */
+export function redactDeep(value, o = {}) {
+  /** @type {Record<string, number>} */
+  const counts = {};
+  const walk = (/** @type any */ v, /** @type string */ key = "") => {
+    if (typeof v === "string") {
+      if (NAME_KEY_RE.test(key) && v.trim() !== "") {
+        counts.name = (counts.name ?? 0) + 1;
+        return "[name]";
+      }
+      const r = redactText(v, o);
+      for (const [k, n] of Object.entries(r.counts)) counts[k] = (counts[k] ?? 0) + n;
+      return r.text;
+    }
+    if (Array.isArray(v)) return v.map((x) => walk(x, key));
+    if (v && typeof v === "object") return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x, k)]));
+    return v;
+  };
+  return { value: walk(value), counts };
+}

package/scripts/eval/replay/lib/sandbox.mjs

@@ -0,0 +1,170 @@
+/**
+ * scripts/eval/replay/lib/sandbox.mjs — what a replay sandbox holds and what
+ * it may do.
+ *
+ * A seat's baseline ran with its own instruction and behaviour surface, so a
+ * replay copies that surface into the sandbox agent root: CLAUDE.md, the
+ * project .claude/ (settings and their hooks, skills, commands, agents),
+ * config/agent.json, agents/, skills/, scripts/hooks/ (project hooks run as
+ * `./scripts/hooks/…`) and policies/ (the send gate reads them). Never copied:
+ * .env files, state/, logs/, memory/, sessions and transcripts, node_modules,
+ * .git, settings.local.json, key and credential files, symlinks, and files over
+ * 1 MB. The seat's absolute path inside a copied text file becomes the sandbox
+ * agent root, and `env` / `apiKeyHelper` are removed from copied settings.
+ *
+ * The permission posture is scoped, not bypass: dontAsk with the read tools and
+ * the replayed org server allowed, and the tools that reach past the stub
+ * (Bash, file writes, the web, spawned work) denied. A fixture may open more
+ * with `sandbox.allowTools`; `sandbox.permissions: "bypass"` is honoured only
+ * when the run passes --allow-bypass.
+ *
+ * Extracted fixtures carry `{{AGENT_ROOT}}` and `{{HOME}}` where the recording
+ * named the seat's root or home; a run fills them with the sandbox's own.
+ *
+ * Pure except `copySeat` and `seatMcpServerNames`, which read the seat and write the sandbox.
+ *
+ * @module scripts/eval/replay/lib/sandbox
+ */
+
+import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+export const PLACEHOLDERS = Object.freeze({ agentRoot: "{{AGENT_ROOT}}", home: "{{HOME}}" });
+
+/** Seat paths copied into the sandbox agent root (files or directories). */
+export const SEAT_INCLUDE = Object.freeze(["CLAUDE.md", "AGENTS.md", ".claude", "config/agent.json", "agents", "skills", "scripts/hooks", "policies"]);
+
+const EXCLUDED_DIRS = new Set(["state", "logs", "memory", "sessions", "transcripts", "projects", "node_modules", ".git", "runtime"]);
+const EXCLUDED_FILE = /^\.env(\..*)?$|^settings\.local\.json$|\.(pem|key|p12|pfx|keychain)$|^id_(rsa|ed25519|ecdsa)|secret|credential|token/i;
+const MAX_COPY_BYTES = 1_000_000;
+
+/** Read tools every replay may use, plus `mcp__<server>`. */
+export const REPLAY_ALLOWED_TOOLS = Object.freeze(["Read", "Glob", "Grep", "TodoWrite", "Skill", "ToolSearch", "ListMcpResources", "ReadMcpResource"]);
+/** Tools that reach past the replay stub; denied unless a fixture opens one. */
+export const REPLAY_DENIED_TOOLS = Object.freeze(["Bash", "Write", "Edit", "MultiEdit", "NotebookEdit", "WebFetch", "WebSearch", "Task", "Agent", "Workflow", "Monitor", "SendMessage"]);
+
+/** Is a seat-relative path (posix separators) excluded from the copy? */
+export function isExcludedSeatPath(/** @type string */ rel) {
+  const parts = rel.split("/");
+  if (parts.slice(0, -1).some((d) => EXCLUDED_DIRS.has(d))) return true;
+  return EXCLUDED_FILE.test(parts.at(-1) ?? "") || EXCLUDED_DIRS.has(parts.at(-1) ?? "");
+}
+
+/** @param {string} p */
+function safeRealpath(p) {
+  try {
+    return realpathSync(p);
+  } catch {
+    return p;
+  }
+}
+
+/** Replace every occurrence of each `from` (longest first) in a string. */
+function replaceRoots(/** @type string */ s, /** @type Array<[string, string]> */ pairs) {
+  let out = s;
+  for (const [from, to] of [...pairs].filter(([f]) => f && f !== "/").sort((a, b) => b[0].length - a[0].length)) out = out.split(from).join(to);
+  return out;
+}
+
+/**
+ * The seat's root and home written as placeholders (extraction). Pure.
+ * @param {string} text @param {{agentRoot:string, home?:string|null, realAgentRoot?:string|null}} o
+ */
+export function toPlaceholders(text, o) {
+  const pairs = /** @type Array<[string,string]> */ ([[o.agentRoot, PLACEHOLDERS.agentRoot]]);
+  if (o.realAgentRoot && o.realAgentRoot !== o.agentRoot) pairs.push([o.realAgentRoot, PLACEHOLDERS.agentRoot]);
+  if (o.home) pairs.push([o.home, PLACEHOLDERS.home]);
+  return replaceRoots(String(text), pairs);
+}
+
+/**
+ * Placeholders filled with a sandbox's paths, in every string of a JSON value. Pure.
+ * @param {unknown} value @param {{agentRoot:string, home:string}} o
+ */
+export function fillPlaceholders(value, o) {
+  if (typeof value === "string") return value.split(PLACEHOLDERS.agentRoot).join(o.agentRoot).split(PLACEHOLDERS.home).join(o.home);
+  if (Array.isArray(value)) return value.map((v) => fillPlaceholders(v, o));
+  if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, fillPlaceholders(v, o)]));
+  return value;
+}
+
+/**
+ * The permission flags for one replay. Pure.
+ * @param {{mcp:{server:string}, sandbox?:{permissions?:string, allowTools?:string[]}}} task
+ * @param {{allowBypass?:boolean}} [o]
+ * @returns {{posture:'scoped'|'bypass', note:string|null, args:string[]}}
+ */
+export function replayPermissions(task, o = {}) {
+  const sb = task.sandbox ?? {};
+  if (sb.permissions === "bypass" && o.allowBypass) return { posture: "bypass", note: "the fixture opted in and the run passed --allow-bypass", args: ["--dangerously-skip-permissions"] };
+  const extra = Array.isArray(sb.allowTools) ? sb.allowTools.map(String) : [];
+  const opened = new Set(extra.map((r) => r.replace(/\(.*$/s, "")));
+  const deny = REPLAY_DENIED_TOOLS.filter((t) => !opened.has(t));
+  return {
+    posture: "scoped",
+    note: sb.permissions === "bypass" ? "the fixture asked for bypass; refused without --allow-bypass" : null,
+    args: ["--permission-mode", "dontAsk", "--allowedTools", [...REPLAY_ALLOWED_TOOLS, `mcp__${task.mcp.server}`, ...extra].join(","), ...(deny.length ? ["--disallowedTools", deny.join(",")] : [])],
+  };
+}
+
+/** The seat's .mcp.json server names, or []. @param {string} seatRoot */
+export function seatMcpServerNames(seatRoot) {
+  try {
+    return Object.keys(JSON.parse(readFileSync(path.join(seatRoot, ".mcp.json"), "utf8")).mcpServers ?? {});
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Copy the seat's instruction and behaviour surface into a sandbox agent root.
+ * @param {string} seatRoot
+ * @param {string} sandboxAgent
+ * @returns {{copied:string[], skipped:string[]}} seat-relative paths
+ */
+export function copySeat(seatRoot, sandboxAgent) {
+  const root = path.resolve(seatRoot);
+  const pairs = /** @type Array<[string,string]> */ ([[root, sandboxAgent], [safeRealpath(root), sandboxAgent]]);
+  const copied = [];
+  const skipped = [];
+  const visit = (/** @type string */ rel) => {
+    const abs = path.join(root, rel);
+    let st;
+    try {
+      st = lstatSync(abs);
+    } catch {
+      return;
+    }
+    if (st.isSymbolicLink() || isExcludedSeatPath(rel)) return void skipped.push(rel);
+    if (st.isDirectory()) {
+      for (const name of readdirSync(abs).sort()) visit(`${rel}/${name}`);
+      return;
+    }
+    if (!st.isFile() || st.size > MAX_COPY_BYTES) return void skipped.push(rel);
+    const target = path.join(sandboxAgent, rel);
+    mkdirSync(path.dirname(target), { recursive: true });
+    const buf = readFileSync(abs);
+    if (buf.includes(0)) copyFileSync(abs, target);
+    else {
+      let text = replaceRoots(buf.toString("utf8"), pairs);
+      if (/^\.claude\/settings(\.[^/]*)?\.json$/.test(rel)) text = stripSettingsSecrets(text);
+      writeFileSync(target, text, { mode: st.mode & 0o777 });
+    }
+    copied.push(rel);
+  };
+  for (const inc of SEAT_INCLUDE) if (existsSync(path.join(root, inc))) visit(inc);
+  return { copied, skipped };
+}
+
+/** A settings file without `env` or `apiKeyHelper` (credentials live there). Pure. */
+export function stripSettingsSecrets(/** @type string */ text) {
+  try {
+    const o = JSON.parse(text);
+    if (!o || typeof o !== "object") return text;
+    delete o.env;
+    delete o.apiKeyHelper;
+    return `${JSON.stringify(o, null, 2)}\n`;
+  } catch {
+    return text;
+  }
+}

package/scripts/eval/replay/lib/schema-check.mjs

@@ -0,0 +1,63 @@
+/**
+ * scripts/eval/replay/lib/schema-check.mjs — does a tool call's input satisfy
+ * the tool's input schema, strictly? Pure.
+ *
+ * The engine coerces and repairs inputs before running a tool; the eval must
+ * not. This checks the JSON Schema subset tool schemas use — type (or a list
+ * of types), required, properties, additionalProperties:false, items, enum,
+ * const, anyOf/oneOf — with no coercion: "3" is not a number.
+ *
+ * @module scripts/eval/replay/lib/schema-check
+ */
+
+/** @param {unknown} v */
+function typeOf(v) {
+  if (v === null) return "null";
+  if (Array.isArray(v)) return "array";
+  if (typeof v === "number") return Number.isInteger(v) ? "integer" : "number";
+  return typeof v;
+}
+
+/** @param {string} want @param {string} got */
+const typeMatches = (want, got) => want === got || (want === "number" && got === "integer");
+
+/**
+ * @param {any} schema
+ * @param {unknown} value
+ * @param {string} [at]
+ * @returns {string[]} errors; empty when valid
+ */
+export function checkSchema(schema, value, at = "$") {
+  if (!schema || typeof schema !== "object") return [];
+  const errors = [];
+  const branches = schema.anyOf ?? schema.oneOf;
+  if (Array.isArray(branches) && branches.length > 0) {
+    if (!branches.some((b) => checkSchema(b, value, at).length === 0)) errors.push(`${at}: matches none of the allowed shapes`);
+  }
+  if (schema.type !== undefined) {
+    const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+    const got = typeOf(value);
+    if (!types.some((t) => typeMatches(t, got))) {
+      errors.push(`${at}: expected ${types.join(" or ")}, got ${got}`);
+      return errors;
+    }
+  }
+  if (Array.isArray(schema.enum) && !schema.enum.some((e) => JSON.stringify(e) === JSON.stringify(value))) errors.push(`${at}: not one of ${JSON.stringify(schema.enum)}`);
+  if (schema.const !== undefined && JSON.stringify(schema.const) !== JSON.stringify(value)) errors.push(`${at}: must be ${JSON.stringify(schema.const)}`);
+  if (typeOf(value) === "object") {
+    const obj = /** @type {Record<string, unknown>} */ (value);
+    for (const key of Array.isArray(schema.required) ? schema.required : []) {
+      if (obj[key] === undefined) errors.push(`${at}.${key}: required`);
+    }
+    const props = schema.properties && typeof schema.properties === "object" ? schema.properties : {};
+    for (const [key, v] of Object.entries(obj)) {
+      if (Object.hasOwn(props, key)) errors.push(...checkSchema(props[key], v, `${at}.${key}`));
+      else if (schema.additionalProperties === false) errors.push(`${at}.${key}: not allowed`);
+      else if (schema.additionalProperties && typeof schema.additionalProperties === "object") errors.push(...checkSchema(schema.additionalProperties, v, `${at}.${key}`));
+    }
+  }
+  if (Array.isArray(value) && schema.items && typeof schema.items === "object") {
+    value.forEach((item, i) => errors.push(...checkSchema(schema.items, item, `${at}[${i}]`)));
+  }
+  return errors;
+}