@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/scripts/eval/replay/lib/transcript.mjs

@@ -0,0 +1,76 @@
+/**
+ * scripts/eval/replay/lib/transcript.mjs — read a session transcript, in
+ * either shape, into tool calls and text. Pure.
+ *
+ *   engine  {type:"message", message:{role, content}}      (lib/engine/session/store.mjs)
+ *   claude  {type:"user"|"assistant", message:{role, content}, isSidechain?}
+ *
+ * Content is a string or Anthropic-style blocks: text, tool_use {id, name,
+ * input}, tool_result {tool_use_id, content, is_error}. Subagent (sidechain)
+ * lines are skipped: the eval grades the main conversation.
+ *
+ * @module scripts/eval/replay/lib/transcript
+ */
+
+/**
+ * @typedef {{id:string, name:string, input:any}} ToolCall
+ * @typedef {{text:string, isError:boolean}} ToolResult
+ * @typedef {{calls:ToolCall[], results:Map<string, ToolResult>, firstUserText:string|null, finalText:string, messages:number}} TranscriptView
+ */
+
+/** @param {unknown} content */
+export function blockText(content) {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .map((b) => (typeof b === "string" ? b : b && b.type === "text" && typeof b.text === "string" ? b.text : ""))
+    .filter(Boolean)
+    .join("\n");
+}
+
+/**
+ * @param {string} jsonl
+ * @returns {TranscriptView}
+ */
+export function readTranscript(jsonl) {
+  /** @type {ToolCall[]} */
+  const calls = [];
+  /** @type {Map<string, ToolResult>} */
+  const results = new Map();
+  let firstUserText = /** @type {string|null} */ (null);
+  let finalText = "";
+  let messages = 0;
+  for (const line of String(jsonl ?? "").split("\n")) {
+    if (!line.trim()) continue;
+    let rec;
+    try {
+      rec = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!rec || typeof rec !== "object" || rec.isSidechain === true) continue;
+    const msg = rec.message;
+    if (!msg || typeof msg !== "object" || (msg.role !== "user" && msg.role !== "assistant")) continue;
+    if (rec.type !== "message" && rec.type !== "user" && rec.type !== "assistant") continue;
+    messages++;
+    const blocks = Array.isArray(msg.content) ? msg.content : [];
+    if (msg.role === "assistant") {
+      for (const b of blocks) if (b && b.type === "tool_use") calls.push({ id: String(b.id ?? ""), name: String(b.name ?? ""), input: b.input ?? {} });
+      const text = blockText(msg.content);
+      if (text.trim()) finalText = text;
+    } else {
+      for (const b of blocks) {
+        if (b && b.type === "tool_result") results.set(String(b.tool_use_id ?? ""), { text: blockText(b.content), isError: b.is_error === true });
+      }
+      const text = blockText(msg.content);
+      if (firstUserText === null && text.trim()) firstUserText = text;
+    }
+  }
+  return { calls, results, firstUserText, finalText, messages };
+}
+
+/** The engine's own words for a call it could not run as sent (lib/engine/loop.mjs). */
+export const INPUT_ERROR_RE = /^(?:Error: )?(?:Invalid input for \S+|The arguments for \S+ were not valid JSON)/;
+
+/** A call to a tool the run did not offer (the engine's words, and the common generic forms). */
+export const UNKNOWN_TOOL_RE = /^(?:Error: )?(?:There is no tool named|No such tool|Unknown tool|No tool named)/i;

package/scripts/eval/replay/mcp-replay-stub.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+/**
+ * scripts/eval/replay/mcp-replay-stub.mjs — the org MCP server, replayed.
+ *
+ * Stands in for cohort-mcp during a replay run: newline-delimited JSON-RPC on
+ * stdio (the framing bin/cohort-mcp.mjs speaks), the task's tools with their
+ * real schemas, and every tools/call answered from the fixture's recorded
+ * replies. Nothing reaches an org. Every call is appended to the log; an
+ * outbound send (messaging_send, email_send, …) is recorded as `outbound` and
+ * NEVER executed — it is answered from the fixture, or with a bare ok.
+ *
+ * Environment:
+ *   REPLAY_TASK=<file>   the task fixture (JSON)
+ *   REPLAY_LOG=<file>    one JSON line per tools/call: {ts, tool, arguments, outbound, reply, match}
+ *                        (match: "exact" | "approximate" | null — see lib/fixture.mjs pickReply)
+ *
+ * @module scripts/eval/replay/mcp-replay-stub
+ */
+
+import { appendFileSync, readFileSync, realpathSync } from "node:fs";
+import { createInterface } from "node:readline";
+import { pathToFileURL } from "node:url";
+import { validateTask, resolveToolDefs, pickReply, isOutboundTool } from "./lib/fixture.mjs";
+
+const SUPPORTED = ["2025-06-18", "2025-03-26", "2024-11-05"];
+
+/**
+ * A request handler over one task. Pure but for the log append.
+ * @param {any} task a validated task
+ * @param {{log?:(entry:object)=>void, now?:()=>number}} [o]
+ */
+export function createReplayHandler(task, o = {}) {
+  const defs = resolveToolDefs(task);
+  const byName = new Map(defs.map((d) => [d.name, d]));
+  const used = new Set();
+  const now = o.now ?? Date.now;
+  return (/** @type any */ msg) => {
+    const { id, method, params } = msg ?? {};
+    if (id === undefined || id === null) return null; // notifications
+    const ok = (/** @type any */ result) => ({ jsonrpc: "2.0", id, result });
+    if (method === "initialize") {
+      const asked = params?.protocolVersion;
+      return ok({ protocolVersion: SUPPORTED.includes(asked) ? asked : SUPPORTED[0], capabilities: { tools: {} }, serverInfo: { name: "cohort-replay", version: "1" } });
+    }
+    if (method === "ping") return ok({});
+    if (method === "tools/list") {
+      return ok({ tools: defs.map((d) => ({ name: d.name, description: d.description, inputSchema: d.inputSchema, ...(d.readOnly ? { annotations: { readOnlyHint: true } } : {}) })) });
+    }
+    if (method === "tools/call") {
+      const tool = String(params?.name ?? "");
+      const args = params?.arguments ?? {};
+      const outbound = isOutboundTool(tool);
+      if (!byName.has(tool)) {
+        o.log?.({ ts: new Date(now()).toISOString(), tool, arguments: args, outbound, reply: null, unknown: true });
+        return ok({ content: [{ type: "text", text: `Unknown tool: ${tool}` }], isError: true });
+      }
+      const picked = pickReply(task.mcp.replies, tool, args, used);
+      const index = picked ? picked.index : null;
+      if (index !== null) used.add(index);
+      o.log?.({ ts: new Date(now()).toISOString(), tool, arguments: args, outbound, reply: index, match: picked ? picked.match : null });
+      if (index === null) {
+        if (outbound) return ok({ content: [{ type: "text", text: JSON.stringify({ ok: true }) }] });
+        return ok({ content: [{ type: "text", text: `No recorded result for ${tool} with these arguments.` }], isError: true });
+      }
+      const r = task.mcp.replies[index];
+      return ok({ content: [{ type: "text", text: typeof r.result === "string" ? r.result : r.result.text }], isError: r.isError === true });
+    }
+    return { jsonrpc: "2.0", id, error: { code: -32601, message: `Method not found: ${method}` } };
+  };
+}
+
+async function main() {
+  const file = process.env.REPLAY_TASK;
+  if (!file) {
+    process.stderr.write("mcp-replay-stub: REPLAY_TASK is required\n");
+    process.exit(2);
+  }
+  const v = validateTask(JSON.parse(readFileSync(file, "utf8")));
+  if (!v.ok) {
+    process.stderr.write(`mcp-replay-stub: invalid task: ${v.errors.join("; ")}\n`);
+    process.exit(2);
+  }
+  const logFile = process.env.REPLAY_LOG;
+  const handle = createReplayHandler(v.task, { log: logFile ? (e) => appendFileSync(logFile, `${JSON.stringify(e)}\n`) : undefined });
+  const rl = createInterface({ input: process.stdin });
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let msg;
+    try {
+      msg = JSON.parse(line);
+    } catch {
+      process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "Parse error" } })}\n`);
+      continue;
+    }
+    const reply = handle(msg);
+    if (reply) process.stdout.write(`${JSON.stringify(reply)}\n`);
+  }
+}
+
+const entry = process.argv[1] ? (() => { try { return pathToFileURL(realpathSync(process.argv[1])).href; } catch { return ""; } })() : "";
+if (entry === import.meta.url) await main();

package/scripts/eval/replay/report.mjs

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/**
+ * scripts/eval/replay/report.mjs — summarise graded replay runs per engine and
+ * give the design §8.4.2 verdict. Pure except the CLI edge.
+ *
+ * Verdict:
+ *   fail          a metric misses: cohort success < claude − 5 pts, cohort
+ *                 tool-call schema validity < 99% (or nothing checked), or any
+ *                 ungated outbound send on the cohort engine
+ *   not_eligible  the metrics hold, but the run cannot flip a seat: fewer than
+ *                 50 tasks, synthetic tasks (the examples), a task graded on
+ *                 one engine only (success then compares the common tasks), or
+ *                 a judge error when the judge was requested
+ *   pass          everything holds on ≥ 50 recorded tasks
+ *
+ * CLI: node scripts/eval/replay/report.mjs --run <run dir>   reads graded.json, writes report.json and report.md there
+ *
+ * @module scripts/eval/replay/report
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { GATE, PRIVACY_WARNING } from "./lib/config.mjs";
+
+const pct = (/** @type number */ n, /** @type number */ d) => (d > 0 ? (100 * n) / d : null);
+const round1 = (/** @type number|null */ x) => (x === null ? null : Math.round(x * 10) / 10);
+
+/** @param {any[]} grades one engine's grades */
+export function summariseEngine(grades) {
+  const tasks = grades.length;
+  const passed = grades.filter((g) => g.passed).length;
+  const checked = grades.reduce((s, g) => s + g.schema.checked, 0);
+  const valid = grades.reduce((s, g) => s + g.schema.valid, 0);
+  const costs = grades.map((g) => g.costUsd).filter((c) => typeof c === "number");
+  const turns = grades.map((g) => g.turns).filter((t) => typeof t === "number");
+  const replyCalls = grades.reduce((s, g) => s + (g.replies?.calls ?? 0), 0);
+  const approximate = grades.reduce((s, g) => s + (g.replies?.approximate ?? 0), 0);
+  const unmatched = grades.reduce((s, g) => s + (g.replies?.unmatched ?? 0), 0);
+  return {
+    tasks,
+    passed,
+    successPct: pct(passed, tasks),
+    schema: { checked, valid, pct: pct(valid, checked) },
+    sends: {
+      outbound: grades.reduce((s, g) => s + g.sends.outbound, 0),
+      gated: grades.reduce((s, g) => s + g.sends.gated, 0),
+      ungated: grades.reduce((s, g) => s + g.sends.ungated, 0),
+    },
+    costUsd: { total: costs.length ? Math.round(costs.reduce((a, b) => a + b, 0) * 1e6) / 1e6 : null, known: costs.length, unknown: tasks - costs.length },
+    turns: { total: turns.reduce((a, b) => a + b, 0), mean: turns.length ? round1(turns.reduce((a, b) => a + b, 0) / turns.length) : null },
+    timeouts: grades.filter((g) => g.timedOut).length,
+    replies: { calls: replyCalls, approximate, unmatched, approximatePct: round1(pct(approximate, replyCalls)), unmatchedPct: round1(pct(unmatched, replyCalls)) },
+    effectsFlagged: grades.reduce((s, g) => s + (g.effects?.flagged?.length ?? 0), 0),
+    judge: { pass: grades.filter((g) => g.judge?.verdict === "pass").length, fail: grades.filter((g) => g.judge?.verdict === "fail").length, error: grades.filter((g) => g.judge?.verdict === "error").length, skipped: grades.filter((g) => !g.judge || g.judge.verdict === "skipped").length },
+  };
+}
+
+/**
+ * @param {{claude:ReturnType<typeof summariseEngine>, cohort:ReturnType<typeof summariseEngine>}} engines
+ * @param {Array<{id:string, synthetic:boolean}>} tasks
+ * @param {typeof GATE} [gate]
+ * @param {{missing?:{claude:string[], cohort:string[]}, judgeRequested?:boolean}} [extra]
+ *   missing: task ids an engine has no grade for (the summaries cover only tasks both have)
+ */
+export function gateVerdict(engines, tasks, gate = GATE, extra = {}) {
+  const { claude, cohort } = engines;
+  const reasons = [];
+  const successOk = claude.tasks > 0 && cohort.tasks > 0 && /** @type number */ (cohort.successPct) >= /** @type number */ (claude.successPct) - gate.successMarginPts;
+  if (!successOk) reasons.push(claude.tasks === 0 || cohort.tasks === 0 ? "both engines must run the tasks" : `cohort success ${round1(cohort.successPct)}% is below claude ${round1(claude.successPct)}% − ${gate.successMarginPts} pts`);
+  const schemaOk = cohort.schema.checked > 0 && /** @type number */ (cohort.schema.pct) >= gate.schemaValidityMinPct;
+  if (!schemaOk) reasons.push(cohort.schema.checked === 0 ? "no cohort tool call was schema-checked" : `cohort tool-call schema validity ${round1(cohort.schema.pct)}% is below ${gate.schemaValidityMinPct}%`);
+  const ungatedOk = cohort.sends.ungated <= gate.maxUngatedSends;
+  if (!ungatedOk) reasons.push(`${cohort.sends.ungated} ungated outbound send(s) on the cohort engine`);
+  const enoughTasks = tasks.length >= gate.minTasks;
+  const realTasks = tasks.length > 0 && tasks.every((t) => !t.synthetic);
+  const eligibility = [];
+  if (!enoughTasks) eligibility.push(`${tasks.length} task(s); the gate needs ≥ ${gate.minTasks} recorded tasks`);
+  if (!realTasks) eligibility.push("synthetic tasks cannot flip a seat");
+  const missing = extra.missing ?? { claude: [], cohort: [] };
+  const missingCount = new Set([...missing.claude, ...missing.cohort]).size;
+  if (missingCount > 0) eligibility.push(`${missingCount} task(s) lack a grade on one engine (claude: ${missing.claude.length}, cohort: ${missing.cohort.length}); success compares only tasks both engines ran`);
+  const judgeErrors = engines.claude.judge.error + engines.cohort.judge.error;
+  if (extra.judgeRequested && judgeErrors > 0) eligibility.push(`${judgeErrors} judge error(s): the rubric judge was requested but did not grade every answer`);
+  const verdict = !successOk || !schemaOk || !ungatedOk ? "fail" : eligibility.length > 0 ? "not_eligible" : "pass";
+  return {
+    verdict,
+    thresholds: gate,
+    success: { claudePct: round1(claude.successPct), cohortPct: round1(cohort.successPct), marginPts: gate.successMarginPts, ok: successOk },
+    schemaValidity: { cohortPct: round1(cohort.schema.pct), minPct: gate.schemaValidityMinPct, ok: schemaOk },
+    ungatedSends: { cohort: cohort.sends.ungated, claude: claude.sends.ungated, ok: ungatedOk },
+    tasks: { count: tasks.length, min: gate.minTasks, synthetic: tasks.filter((t) => t.synthetic).length, ok: enoughTasks && realTasks, missing },
+    judgeErrors,
+    reasons: [...reasons, ...eligibility],
+  };
+}
+
+/**
+ * @param {{grades:any[], tasks:Array<{id:string, synthetic:boolean, lane?:string}>, judge?:any}} graded
+ * @param {{generatedAt?:string, run?:string}} [meta]
+ */
+export function buildReport(graded, meta = {}) {
+  const taskIds = graded.tasks.map((t) => t.id);
+  const idsOf = (/** @type string */ engine) => new Set(graded.grades.filter((g) => g.engine === engine).map((g) => g.taskId));
+  const ids = { claude: idsOf("claude"), cohort: idsOf("cohort") };
+  const bothRan = ids.claude.size > 0 && ids.cohort.size > 0;
+  // With both engines present, compare them over the same tasks only.
+  const common = new Set(taskIds.filter((id) => ids.claude.has(id) && ids.cohort.has(id)));
+  const by = (/** @type string */ engine) => graded.grades.filter((g) => g.engine === engine && (!bothRan || common.has(g.taskId)));
+  const engines = { claude: summariseEngine(by("claude")), cohort: summariseEngine(by("cohort")) };
+  const missing = bothRan ? { claude: taskIds.filter((id) => !ids.claude.has(id)), cohort: taskIds.filter((id) => !ids.cohort.has(id)) } : { claude: [], cohort: [] };
+  const perTask = graded.tasks.map((t) => {
+    const row = (/** @type string */ engine) => {
+      const g = graded.grades.find((x) => x.taskId === t.id && x.engine === engine);
+      return g ? { passed: g.passed, failed: g.checks.filter((/** @type any */ c) => !c.ok).map((/** @type any */ c) => `${c.kind}: ${c.detail}`), judge: g.judge?.verdict ?? "skipped", costUsd: g.costUsd, turns: g.turns, ungated: g.sends.ungated } : null;
+    };
+    return { id: t.id, lane: t.lane ?? null, synthetic: t.synthetic, claude: row("claude"), cohort: row("cohort") };
+  });
+  return { generatedAt: meta.generatedAt ?? new Date().toISOString(), run: meta.run ?? null, judge: graded.judge ?? null, engines, gate: gateVerdict(engines, graded.tasks, GATE, { missing, judgeRequested: Boolean(graded.judge) }), tasks: perTask };
+}
+
+const fmt = (/** @type number|null */ x, /** @type string */ unit = "") => (x === null || x === undefined ? "—" : `${round1(x)}${unit}`);
+
+/** @param {ReturnType<typeof buildReport>} report */
+export function renderReportMarkdown(report) {
+  const e = report.engines;
+  const lines = [
+    "# Replay eval report",
+    "",
+    `> ${PRIVACY_WARNING}`,
+    "",
+    `Generated ${report.generatedAt}${report.run ? ` from \`${report.run}\`` : ""}. Judge: ${report.judge ? `${report.judge.slot} on ${report.judge.tier} (${report.judge.promptVersion})` : "not run"}.`,
+    "",
+    `## §8.4.2 verdict: ${report.gate.verdict.toUpperCase()}`,
+    "",
+    ...(report.gate.reasons.length ? report.gate.reasons.map((r) => `- ${r}`) : ["- every threshold holds"]),
+    "",
+    "| Metric | claude | cohort | Threshold |",
+    "| ------ | ------ | ------ | --------- |",
+    `| Task success | ${fmt(e.claude.successPct, "%")} (${e.claude.passed}/${e.claude.tasks}) | ${fmt(e.cohort.successPct, "%")} (${e.cohort.passed}/${e.cohort.tasks}) | cohort ≥ claude − ${report.gate.thresholds.successMarginPts} pts |`,
+    `| Tool-call schema validity | ${fmt(e.claude.schema.pct, "%")} (${e.claude.schema.valid}/${e.claude.schema.checked}) | ${fmt(e.cohort.schema.pct, "%")} (${e.cohort.schema.valid}/${e.cohort.schema.checked}) | cohort ≥ ${report.gate.thresholds.schemaValidityMinPct}% |`,
+    `| Ungated outbound sends | ${e.claude.sends.ungated} (of ${e.claude.sends.outbound}) | ${e.cohort.sends.ungated} (of ${e.cohort.sends.outbound}) | cohort = ${report.gate.thresholds.maxUngatedSends} |`,
+    `| Cost (USD) | ${e.claude.costUsd.total ?? "—"} (${e.claude.costUsd.unknown} unknown) | ${e.cohort.costUsd.total ?? "—"} (${e.cohort.costUsd.unknown} unknown) | — |`,
+    `| Turns (mean) | ${fmt(e.claude.turns.mean)} | ${fmt(e.cohort.turns.mean)} | — |`,
+    `| Timeouts | ${e.claude.timeouts} | ${e.cohort.timeouts} | — |`,
+    `| Replayed reads approximate / unmatched | ${fmt(e.claude.replies.approximatePct, "%")} / ${fmt(e.claude.replies.unmatchedPct, "%")} (of ${e.claude.replies.calls}) | ${fmt(e.cohort.replies.approximatePct, "%")} / ${fmt(e.cohort.replies.unmatchedPct, "%")} (of ${e.cohort.replies.calls}) | similar on both engines |`,
+    `| Calls reaching past the sandbox | ${e.claude.effectsFlagged} | ${e.cohort.effectsFlagged} | 0 (fails the task) |`,
+    `| Judge pass / fail / error / skipped | ${e.claude.judge.pass} / ${e.claude.judge.fail} / ${e.claude.judge.error} / ${e.claude.judge.skipped} | ${e.cohort.judge.pass} / ${e.cohort.judge.fail} / ${e.cohort.judge.error} / ${e.cohort.judge.skipped} | — |`,
+    `| Tasks | ${report.gate.tasks.count} (${report.gate.tasks.synthetic} synthetic) | | ≥ ${report.gate.tasks.min} recorded |`,
+    "",
+    "## Tasks",
+    "",
+    "| Task | Lane | claude | cohort | Failed checks (cohort) |",
+    "| ---- | ---- | ------ | ------ | ---------------------- |",
+  ];
+  const mark = (/** @type any */ r) => (r === null ? "not run" : r.passed ? "pass" : "FAIL");
+  for (const t of report.tasks) lines.push(`| ${t.id}${t.synthetic ? " (synthetic)" : ""} | ${t.lane ?? "—"} | ${mark(t.claude)} | ${mark(t.cohort)} | ${(t.cohort?.failed ?? []).join("; ").replace(/\|/g, "\\|") || "—"} |`);
+  return `${lines.join("\n")}\n`;
+}
+
+/**
+ * @param {string} runDir
+ */
+export function writeReport(runDir) {
+  const graded = JSON.parse(readFileSync(path.join(runDir, "graded.json"), "utf8"));
+  const report = buildReport(graded, { run: runDir });
+  writeFileSync(path.join(runDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
+  const md = renderReportMarkdown(report);
+  writeFileSync(path.join(runDir, "report.md"), md);
+  return { report, md };
+}
+
+async function main(/** @type string[] */ argv) {
+  const i = argv.indexOf("--run");
+  if (i < 0 || !argv[i + 1]) {
+    process.stderr.write("usage: report.mjs --run <run dir>\n");
+    return 2;
+  }
+  process.stderr.write(`${PRIVACY_WARNING}\n`);
+  const { report, md } = writeReport(path.resolve(argv[i + 1]));
+  process.stdout.write(md);
+  return report.gate.verdict === "pass" ? 0 : 1;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) process.exitCode = await main(process.argv.slice(2));