@runcofactor/cli 0.52.0 → 0.53.0

package/dist/eval-SO5TPD37.js

@@ -0,0 +1,1177 @@
+import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
+import {
+  resolveApplicationId
+} from "./chunk-JEHHEMUL.js";
+import "./chunk-RVD6J6IZ.js";
+import {
+  apiRequest,
+  getArgValues,
+  getPositionalArg,
+  getPositionalArgs,
+  getScalarArgValue,
+  readStdin
+} from "./chunk-VKZOHJCE.js";
+import "./chunk-S3Q7MDCS.js";
+import "./chunk-4UZN6TVT.js";
+import {
+  GLOBAL_VALUE_FLAGS,
+  apiFetch,
+  hasFlag,
+  resolveConfig
+} from "./chunk-WRBU6TX7.js";
+import {
+  writeRawOutput
+} from "./chunk-MJVUMVLO.js";
+import "./chunk-TOWZAG3I.js";
+
+// src/eval.ts
+import { mkdir, readFile as readFile3, readdir as readdir2, writeFile } from "fs/promises";
+import { join as join2 } from "path";
+
+// ../types/src/agent-eval-template.ts
+import { z } from "zod";
+var AGENT_EVAL_MAX_BUNDLE_BYTES = 512 * 1024 ** 2;
+var AGENT_EVAL_MAX_TOTAL_BUNDLE_BYTES = 2 * 1024 ** 3;
+var ModelSchema = z.strictObject({
+  provider: z.enum(["anthropic", "azure"]),
+  model: z.string().min(1)
+});
+var ScriptGrader = z.strictObject({
+  id: z.string().regex(/^[a-z0-9-]+$/),
+  kind: z.literal("script"),
+  command: z.string().min(1)
+});
+var JudgeGrader = z.strictObject({
+  id: z.string().regex(/^[a-z0-9-]+$/),
+  kind: z.literal("ai-judge"),
+  rubric: z.string().min(20),
+  scale: z.enum(["binary", "1-5"]),
+  passAt: z.number().int().min(2).max(5).optional(),
+  // required when scale is '1-5'
+  judge: ModelSchema
+});
+var AgentEvalTemplateSchema = z.strictObject({
+  version: z.literal(1),
+  name: z.string().regex(/^[a-z0-9-]{2,40}$/).refine((n) => n !== "runs" && n !== "compare"),
+  // reserved route segments
+  description: z.string().min(1).max(500),
+  proves: z.strictObject({ capabilities: z.array(z.string().regex(/^CAP-/)).min(1) }),
+  // resolved within the application given at push
+  runbook: z.string().min(1),
+  // path relative to this file
+  mode: z.enum(["single-turn", "multi-turn"]).default("single-turn"),
+  run: z.string().min(1),
+  // shell command, cwd = repo root of the commit; multi-turn: the agent process, JSONL on stdin/stdout
+  simulator: ModelSchema.optional(),
+  // required when mode is multi-turn: pins the simulated-user model
+  input: z.strictObject({
+    description: z.string().min(1),
+    require: z.array(z.string()).default([])
+  }),
+  agent: z.strictObject({
+    models: z.record(z.string().regex(/^[a-z0-9-]+$/), ModelSchema).refine((m) => Object.keys(m).length >= 1),
+    default: z.string()
+    // must be a key of models
+  }),
+  graders: z.array(z.discriminatedUnion("kind", [ScriptGrader, JudgeGrader])).min(1),
+  repeats: z.number().int().min(1).max(7).default(3),
+  passAtLeast: z.number().int().min(1).default(2),
+  // <= repeats
+  passBar: z.number().min(0.1).max(1),
+  // minimum case pass rate for the eval to pass
+  attemptSeconds: z.number().int().min(30).max(1700)
+  // grader commands get a fixed 120 s
+}).superRefine((template, ctx) => {
+  if (!Object.hasOwn(template.agent.models, template.agent.default)) {
+    ctx.addIssue({
+      code: "custom",
+      path: ["agent", "default"],
+      message: "Default must name an agent model alias"
+    });
+  }
+  if (template.passAtLeast > template.repeats) {
+    ctx.addIssue({
+      code: "custom",
+      path: ["passAtLeast"],
+      message: "passAtLeast must not exceed repeats"
+    });
+  }
+  const ids = /* @__PURE__ */ new Set();
+  template.graders.forEach((grader, index) => {
+    if (ids.has(grader.id)) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["graders", index, "id"],
+        message: "Grader ids must be unique"
+      });
+    }
+    ids.add(grader.id);
+    if (grader.kind === "ai-judge" && grader.scale === "1-5" && grader.passAt === void 0) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["graders", index, "passAt"],
+        message: "A 1-5 judge requires passAt"
+      });
+    }
+  });
+  if (template.mode === "multi-turn" !== (template.simulator !== void 0)) {
+    ctx.addIssue({
+      code: "custom",
+      path: ["simulator"],
+      message: "A simulator is required exactly when mode is multi-turn"
+    });
+  }
+});
+var EvalConversationSchema = z.discriminatedUnion("kind", [
+  z.strictObject({ kind: z.literal("scripted"), turns: z.array(z.string().min(1)).min(1) }),
+  z.strictObject({
+    kind: z.literal("simulated"),
+    persona: z.string().min(1),
+    goal: z.string().min(1),
+    facts: z.array(z.string()),
+    stopWhen: z.string().min(1),
+    maxTurns: z.number().int().min(2).max(30)
+  })
+]);
+
+// src/eval-api.ts
+var EVALS = "/org/agent-evals";
+var isRecord = (value) => typeof value === "object" && value !== null;
+function describeDetails(details) {
+  if (typeof details === "string") return `
+  ${details}`;
+  if (!Array.isArray(details)) return "";
+  const lines = details.flatMap((item) => {
+    if (typeof item === "string") return [item];
+    if (!isRecord(item) || typeof item.message !== "string") return [];
+    const path = Array.isArray(item.path) ? item.path.map(String).join(".") : "";
+    return [`${path ? `${path}: ` : ""}${item.message}`];
+  });
+  return lines.length ? `
+${lines.map((line) => `  ${line}`).join("\n")}` : "";
+}
+function describeParams(response) {
+  return ["reason", "alias", "capability", "secret"].flatMap((key) => typeof response[key] === "string" ? [`
+  ${key}: ${response[key]}`] : []).join("");
+}
+function describeFailure(response, method, path) {
+  const message = `${response.code ? `${response.code}: ` : ""}${response.error ?? `${method} ${path} failed`}${describeParams(response)}${describeDetails(response.details)}${describeDetails(response.issues)}`;
+  return response.code === "API_KEY_INSUFFICIENT_SCOPE" && method === "POST" && path.endsWith("/runs") ? `${message}
+  eval run needs an API key with sandbox:create and sandbox:exec; sandbox:* does not grant sandbox:exec.` : message;
+}
+function failureText(failure, method, path) {
+  try {
+    const parsed = JSON.parse(failure.body ?? "");
+    if (isRecord(parsed) && typeof parsed.error === "string")
+      return describeFailure(parsed, method, path);
+  } catch {
+  }
+  return failure.message;
+}
+async function request(apiUrl, token, method, path, body) {
+  const response = await apiRequest(apiUrl, token, method, path, body);
+  if (!response.ok || response.data === void 0) throw new Error(describeFailure(response, method, path));
+  return response.data;
+}
+var authHeaders = (token) => ({ Authorization: `Bearer ${token}`, "User-Agent": "Cofactor-CLI" });
+async function requestText(apiUrl, token, path) {
+  const result = await apiFetch(`${apiUrl}${path}`, { headers: authHeaders(token), token });
+  if (!result.ok) throw new Error(failureText(result.failure, "GET", path));
+  return result.response.text();
+}
+async function requestTolerant(apiUrl, token, path) {
+  const result = await apiFetch(`${apiUrl}${path}`, { headers: authHeaders(token), token });
+  if (!result.ok) {
+    const { failure } = result;
+    if (failure.kind === "network" || failure.status !== null && (failure.status >= 500 || failure.status === 429))
+      return { transient: failure.message };
+    throw new Error(failureText(failure, "GET", path));
+  }
+  const body = await result.response.json();
+  if (!body.ok || body.data === void 0) throw new Error(describeFailure(body, "GET", path));
+  return { data: body.data };
+}
+async function requestAll(apiUrl, token, path) {
+  const items = [];
+  for (let page = 1; ; page++) {
+    const response = await apiRequest(
+      apiUrl,
+      token,
+      "GET",
+      `${path}?pageSize=100&page=${page}`
+    );
+    if (!response.ok || !response.data) throw new Error(describeFailure(response, "GET", path));
+    items.push(...response.data);
+    if (page >= (response.pagination?.totalPages ?? 1)) return items;
+  }
+}
+var getEval = (apiUrl, token, name) => request(apiUrl, token, "GET", `${EVALS}/${encodeURIComponent(name)}`);
+
+// src/eval-bundle.ts
+import { createHash } from "crypto";
+import { lstat, readdir, readFile } from "fs/promises";
+import { join } from "path";
+import { gunzipSync, gzipSync } from "zlib";
+var BLOCK = 512;
+var MAX_RAW_BYTES = AGENT_EVAL_MAX_BUNDLE_BYTES;
+var MAX_UNPACKED_BYTES = 4 * AGENT_EVAL_MAX_BUNDLE_BYTES;
+var sha256Hex = (data) => createHash("sha256").update(data).digest("hex");
+function octal(value, width) {
+  return `${value.toString(8).padStart(width - 1, "0")}\0`;
+}
+function splitName(path) {
+  if (Buffer.byteLength(path) <= 100) return { prefix: "", name: path };
+  for (let i = path.indexOf("/"); i !== -1; i = path.indexOf("/", i + 1)) {
+    const prefix = path.slice(0, i);
+    const name = path.slice(i + 1);
+    if (Buffer.byteLength(prefix) <= 155 && Buffer.byteLength(name) <= 100) return { prefix, name };
+  }
+  throw new Error(`Path is too long for a bundle (limit 255 bytes): ${path}`);
+}
+function header(path, size, type, mode) {
+  const { prefix, name } = splitName(path);
+  const block = Buffer.alloc(BLOCK);
+  block.write(name, 0, "utf8");
+  block.write(octal(mode, 8), 100, "ascii");
+  block.write(octal(0, 8), 108, "ascii");
+  block.write(octal(0, 8), 116, "ascii");
+  block.write(octal(size, 12), 124, "ascii");
+  block.write(octal(0, 12), 136, "ascii");
+  block.fill(32, 148, 156);
+  block.write(type, 156, "ascii");
+  block.write("ustar\0", 257, "ascii");
+  block.write("00", 263, "ascii");
+  block.write(prefix, 345, "utf8");
+  const checksum = block.reduce((sum, byte) => sum + byte, 0);
+  block.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
+  return block;
+}
+async function walk(root, real = "", archive = "") {
+  const names = (await readdir(real ? join(root, real) : root)).map((name) => ({ real: name, archive: name.normalize("NFC") })).sort((a, b) => Buffer.compare(Buffer.from(a.archive), Buffer.from(b.archive)));
+  const entries = [];
+  for (const name of names) {
+    const realPath = real ? `${real}/${name.real}` : name.real;
+    const path = archive ? `${archive}/${name.archive}` : name.archive;
+    const absolute = join(root, realPath);
+    const stats = await lstat(absolute);
+    if (stats.isDirectory()) {
+      entries.push({ path, absolute, directory: true, mode: 493, size: 0 });
+      entries.push(...await walk(root, realPath, path));
+    } else if (stats.isFile()) {
+      entries.push({
+        path,
+        absolute,
+        directory: false,
+        mode: stats.mode & 73 ? 493 : 420,
+        size: stats.size
+      });
+    } else {
+      throw new Error(
+        `Cannot bundle ${path}: only regular files and directories are supported (no symlinks or devices).`
+      );
+    }
+  }
+  return entries;
+}
+async function tarDirectory(root) {
+  const entries = await walk(root);
+  const total = entries.reduce((sum, entry) => sum + entry.size, 0);
+  if (total > MAX_RAW_BYTES) {
+    throw new Error(`The input directory holds ${total} bytes; the CLI bundles at most ${MAX_RAW_BYTES}. Prune it.`);
+  }
+  const parts = [];
+  for (const entry of entries) {
+    if (entry.directory) {
+      parts.push(header(`${entry.path}/`, 0, "5", entry.mode));
+      continue;
+    }
+    const content = await readFile(entry.absolute);
+    parts.push(header(entry.path, content.length, "0", entry.mode), content);
+    const padding = (BLOCK - content.length % BLOCK) % BLOCK;
+    if (padding) parts.push(Buffer.alloc(padding));
+  }
+  parts.push(Buffer.alloc(BLOCK * 2));
+  return Buffer.concat(parts);
+}
+function readString(block, start, length) {
+  const slice = block.subarray(start, start + length);
+  const end = slice.indexOf(0);
+  return slice.subarray(0, end === -1 ? length : end).toString("utf8");
+}
+function normalize(path) {
+  return path.replace(/^(\.\/)+/, "").replace(/\/+$/, "");
+}
+function listTarEntries(tar) {
+  const entries = [];
+  let longName = null;
+  let offset = 0;
+  while (offset + BLOCK <= tar.length) {
+    const block = tar.subarray(offset, offset + BLOCK);
+    if (block.every((byte) => byte === 0)) break;
+    const stored = Number.parseInt(readString(block, 148, 8).trim(), 8);
+    const actual = block.reduce((sum, byte, i) => sum + (i >= 148 && i < 156 ? 32 : byte), 0);
+    if (stored !== actual) throw new Error("not a valid tar archive (bad header checksum)");
+    const size = Number.parseInt(readString(block, 124, 12).trim() || "0", 8);
+    const type = String.fromCharCode(block[156] || 48);
+    const body = tar.subarray(offset + BLOCK, offset + BLOCK + size);
+    offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
+    if (type === "L") {
+      longName = readString(body, 0, body.length);
+      continue;
+    }
+    if (type === "x") {
+      const record = body.toString("utf8").match(/\d+ path=([^\n]*)\n/);
+      if (record?.[1]) longName = record[1];
+      continue;
+    }
+    if (type === "g") continue;
+    const prefix = readString(block, 345, 155);
+    if (["1", "2", "3", "4", "6"].includes(type))
+      throw new Error(`the archive holds a link or device entry (${readString(block, 0, 100)}); only files and directories are supported`);
+    const name = longName ?? (prefix ? `${prefix}/${readString(block, 0, 100)}` : readString(block, 0, 100));
+    longName = null;
+    entries.push(normalize(name));
+  }
+  return entries;
+}
+async function loadBundle(input) {
+  let stats;
+  try {
+    stats = await lstat(input);
+  } catch {
+    throw new Error(`--input ${input} does not exist.`);
+  }
+  let tar;
+  if (stats.isDirectory()) {
+    tar = await tarDirectory(input);
+  } else if (stats.isFile() && /\.(tar\.gz|tgz)$/.test(input)) {
+    if (stats.size > AGENT_EVAL_MAX_BUNDLE_BYTES)
+      throw new Error(`${input} is ${stats.size} bytes; a bundle is limited to ${AGENT_EVAL_MAX_BUNDLE_BYTES}. Prune it.`);
+    const gzip = await readFile(input);
+    try {
+      tar = gunzipSync(gzip, { maxOutputLength: MAX_UNPACKED_BYTES });
+    } catch {
+      throw new Error(`--input ${input} is not a valid gzip file, or unpacks to more than ${MAX_UNPACKED_BYTES} bytes.`);
+    }
+  } else {
+    throw new Error("--input must be a directory or a .tar.gz file.");
+  }
+  const archive = gzipSync(tar, { level: 9 });
+  let entries;
+  try {
+    entries = listTarEntries(tar);
+  } catch (error) {
+    throw new Error(`--input ${input}: ${error instanceof Error ? error.message : String(error)}.`);
+  }
+  return {
+    sha256: sha256Hex(tar),
+    gzipSha256: sha256Hex(archive),
+    bytes: archive.length,
+    archive,
+    entries
+  };
+}
+function missingRequiredPaths(entries, required2) {
+  return required2.filter((path) => {
+    const wanted = normalize(path);
+    return !entries.some((entry) => entry === wanted || entry.startsWith(`${wanted}/`));
+  });
+}
+
+// src/eval-run.ts
+import { setTimeout as sleep } from "timers/promises";
+
+// ../types/src/agent-eval-render.ts
+var cell = (value) => value.replaceAll("|", "\\|").replace(/\r?\n/g, " ");
+var percent = (value) => `${Math.round(value * 100)}%`;
+var money = (cents, incomplete = false) => cents === null ? "unpriced" : `$${(cents / 100).toFixed(2)}${incomplete ? "*" : ""}`;
+var minutes = (ms) => (ms / 6e4).toFixed(1);
+var tokenCount = (n) => n >= 1e3 ? `${(n / 1e3).toFixed(n >= 1e5 ? 0 : 1)}k` : `${Math.round(n)}`;
+var tokenPair = (tokens) => `${tokenCount(tokens.input)} / ${tokenCount(tokens.output)}`;
+var meanTaskTokens = (result) => {
+  const cases = result.cases;
+  const mean = (key) => cases.length ? cases.reduce((n, c) => n + c.tokensPerAttempt[key], 0) / cases.length : 0;
+  return {
+    input: mean("input"),
+    output: mean("output"),
+    cacheRead: mean("cacheRead"),
+    cacheCreation: mean("cacheCreation")
+  };
+};
+var verdict = (r) => (r.verdict ?? r.status).toUpperCase();
+function renderAgentEvalRun(result) {
+  const s = result.summary;
+  const taskTokens = meanTaskTokens(result);
+  const warning = result.warnings.length ? ` [WARNING: ${result.warnings.map(cell).join("; ")}]` : "";
+  const multiTurn = s.userSim !== null;
+  const turnsToSuccess = (c) => {
+    const passing = c.attempts.flatMap((a) => a.passed && a.turns !== null ? [a.turns] : []);
+    return passing.length ? (passing.reduce((n, t) => n + t, 0) / passing.length).toFixed(1) : "-";
+  };
+  const lines = [
+    `## Eval ${cell(result.eval)}: ${verdict(result)} (${s.casesPassed}/${s.cases} cases, bar ${percent(result.passBar)}) \u2014 ${cell(result.model.alias)} (${cell(result.model.id)}) @ ${result.commit.slice(0, 7)}, ${result.repeats} repeats${warning}`,
+    "",
+    `| Case | Result | Attempts | Score | ${multiTurn ? "Turns | " : ""}Cost/attempt | Time/attempt | Tokens in/out |`,
+    `|---|---|---|---|${multiTurn ? "---|" : ""}---|---|---|`,
+    ...result.cases.map(
+      (c) => `| ${cell(c.key)} | ${c.passed ? "PASS" : "FAIL"} | ${c.passedAttempts}/${c.attempts.length} | ${c.meanScore.toFixed(1)} | ${multiTurn ? `${turnsToSuccess(c)} | ` : ""}${money(
+        c.costCentsPerAttempt,
+        c.attempts.some((a) => !a.costComplete)
+      )} | ${minutes(c.durationMsPerAttempt)} min | ${tokenPair(c.tokensPerAttempt)} |`
+    ),
+    "",
+    `Per task: ${money(s.costCentsPerTask, s.incompleteCalls > 0)} | ${minutes(s.durationMsPerTask)} min | ${tokenCount(taskTokens.input)} in / ${tokenCount(taskTokens.output)} out. Total: ${money(s.costCents, s.incompleteCalls > 0)} for ${s.attempts} attempts, ${Math.round(result.cases.flatMap((c) => c.attempts).reduce((n, a) => n + a.durationMs, 0) / 6e4)} min of agent time.`
+  ];
+  const judges = [
+    ...new Set(
+      result.cases.flatMap((c) => c.attempts.flatMap((a) => a.grades)).filter((g) => g.kind === "ai-judge").map((g) => g.judgeModel)
+    )
+  ];
+  lines.push(
+    `Judge (${judges.length ? judges.map((m) => cell(m ?? "unknown")).join(", ") : "none"}, pinned): ${s.judge.calls} calls, ${money(s.judge.costCents)}, not included above.`
+  );
+  if (s.userSim) {
+    const models = [
+      ...new Set(
+        result.cases.flatMap(
+          (c) => c.attempts.flatMap((a) => a.calls.filter((call) => call.role === "user-sim").map((call) => call.model))
+        )
+      )
+    ];
+    lines.push(
+      `User-sim (${models.length ? models.map(cell).join(", ") : "none"}, pinned): ${s.userSim.calls} calls, ${money(s.userSim.costCents)}, not included above.`
+    );
+  }
+  const failures = result.cases.flatMap(
+    (c) => c.attempts.flatMap((a) => [
+      ...a.grades.filter((g) => !g.pass).map((g) => `${cell(c.key)} #${a.index}: ${cell(g.grader)}: ${cell(g.reason)}`),
+      ...a.error && !a.grades.some((g) => !g.pass && (g.reason === a.error || g.error === a.error)) ? [`${cell(c.key)} #${a.index}: ${cell(a.error)}`] : []
+    ])
+  );
+  lines.push(`Failed grades: ${failures.length ? failures.join("; ") : "none"}.`);
+  if (result.error) lines.push(`Error: ${cell(result.error)}.`);
+  if (s.incompleteCalls) lines.push("* Cost is a lower bound because agent calls were incomplete.");
+  lines.push(
+    `Warnings: ${result.warnings.length ? result.warnings.map(cell).join("; ") : "none"}. Run: ${result.id}.`
+  );
+  return `${lines.join("\n")}
+`;
+}
+function renderAgentEvalCompare(result) {
+  const lines = [
+    `## Eval compare: ${cell(result.eval)} @ ${result.commit.slice(0, 7)}${result.batchId ? ` (batch ${result.batchId})` : ""}`,
+    "",
+    "| Model | Verdict | Pass rate | Mean score | Cost/task | Time/task | Tokens/task in/out |",
+    "|---|---|---|---|---|---|---|",
+    ...result.runs.map(
+      (r) => `| ${cell(r.model.alias)} (${cell(r.model.id)}) | ${verdict(r)} | ${percent(r.summary.passRate)} | ${r.summary.meanScore.toFixed(1)} | ${money(r.summary.costCentsPerTask, r.summary.incompleteCalls > 0)} | ${minutes(r.summary.durationMsPerTask)} min | ${tokenPair(meanTaskTokens(r))} |`
+    ),
+    "",
+    `| Case | ${result.runs.map((r) => cell(r.model.alias)).join(" | ")} |`,
+    `|---|${result.runs.map(() => "---|").join("")}`,
+    ...result.cases.map(
+      (c) => `| ${cell(c.key)} | ${c.models.map(
+        ({ result: r }) => `${r.passedAttempts}/${r.attempts.length} ${r.passed ? "PASS" : "FAIL"} ${money(
+          r.costCentsPerAttempt,
+          r.attempts.some((a) => !a.costComplete)
+        )} ${minutes(r.durationMsPerAttempt)}m`
+      ).join(" | ")} |`
+    )
+  ];
+  for (const r of result.runs)
+    if (r.warnings.length)
+      lines.push(`Warnings (${cell(r.model.alias)}): ${r.warnings.map(cell).join("; ")}.`);
+  if (result.runs.some((r) => r.summary.incompleteCalls > 0))
+    lines.push("* Cost is a lower bound because agent calls were incomplete.");
+  return `${lines.join("\n")}
+`;
+}
+
+// src/eval-run.ts
+var pollSettings = { intervalMs: 5e3 };
+var WAIT_MS = 8.5 * 60 * 60 * 1e3;
+var MAX_TRANSIENT_FAILURES = 5;
+var FINISHED = ["completed", "failed", "cancelled"];
+function intFlag(argv, flag) {
+  const value = getScalarArgValue(argv, flag);
+  if (value === void 0) return void 0;
+  if (!/^\d+$/.test(value)) throw new Error(`${flag} must be a whole number.`);
+  return Number(value);
+}
+var runPath = (id, transcript = false) => `${EVALS}/runs/${encodeURIComponent(id)}${transcript ? "?transcript=1" : ""}`;
+var describeRun = (run) => `${run.id}  ${run.modelAlias}  ${run.status}${run.verdict ? `  ${run.verdict}` : ""}${run.error ? `  ${run.error}` : ""}`;
+var stub = (run) => ({ id: run.id, status: run.status, verdict: run.verdict, error: run.error });
+function exitCodeFor(runs) {
+  if (runs.some((run) => run.status !== "completed")) return 1;
+  return runs.some((run) => run.verdict !== "pass") ? 3 : 0;
+}
+async function waitForRuns(apiUrl, token, evalName, started) {
+  const ids = `batch ${started.batchId}; runs ${started.runIds.join(", ")}`;
+  let transient = 0;
+  const get = async (path) => {
+    const reply = await requestTolerant(apiUrl, token, path);
+    if ("data" in reply) {
+      transient = 0;
+      return reply.data;
+    }
+    if (++transient >= MAX_TRANSIENT_FAILURES)
+      throw new Error(`Giving up after ${transient} failed polls (${reply.transient}). Still readable: ${ids}.`);
+    return null;
+  };
+  const deadline = Date.now() + WAIT_MS;
+  for (; ; ) {
+    if (Date.now() >= deadline)
+      throw new Error(`Timed out waiting for the runs. Inspect them with cofactor eval get <run-id>: ${ids}.`);
+    const page = await get(`${EVALS}/${encodeURIComponent(evalName)}/runs?limit=100`);
+    if (page) {
+      const statuses = new Map(page.items.map((item) => [item.id, item.status]));
+      const known = await Promise.all(
+        started.runIds.map(async (id) => statuses.get(id) ?? (await get(runPath(id)))?.status)
+      );
+      if (known.every((status) => status !== void 0 && FINISHED.includes(status))) break;
+    }
+    await sleep(pollSettings.intervalMs);
+  }
+  const details = [];
+  for (const id of started.runIds) {
+    let detail = null;
+    while (!(detail = await get(runPath(id)))) await sleep(pollSettings.intervalMs);
+    details.push(detail);
+  }
+  return details;
+}
+async function runEval(argv, name, apiUrl, token) {
+  const json = hasFlag(argv, "--json");
+  const models = getArgValues(argv, "--model");
+  const cases = getArgValues(argv, "--case");
+  const at = getScalarArgValue(argv, "--at");
+  const environment = getScalarArgValue(argv, "--environment");
+  const repeats = intFlag(argv, "--repeats");
+  const parallel = intFlag(argv, "--parallel");
+  const evaluation = await getEval(apiUrl, token, name);
+  const started = await request(
+    apiUrl,
+    token,
+    "POST",
+    `${EVALS}/${encodeURIComponent(evaluation.id)}/runs`,
+    {
+      ...models.length ? { models } : {},
+      ...environment ? { environment } : {},
+      ...at ? { at } : {},
+      ...cases.length ? { cases } : {},
+      ...repeats !== void 0 ? { repeats } : {},
+      ...parallel !== void 0 ? { parallel } : {}
+    }
+  );
+  if (hasFlag(argv, "--no-wait")) {
+    writeRawOutput(
+      json ? JSON.stringify(started, null, 2) : `Started batch ${started.batchId}
+${started.runIds.join("\n")}
+Read results with: cofactor eval get <run-id>  |  cofactor eval compare --batch ${started.batchId}`
+    );
+    return 0;
+  }
+  console.error(
+    `Started batch ${started.batchId}; runs ${started.runIds.join(", ")}. If this is interrupted, read results with cofactor eval get <run-id> or cofactor eval compare --batch ${started.batchId}.`
+  );
+  const details = await waitForRuns(apiUrl, token, name, started);
+  const results = details.flatMap((run) => run.result ? [run.result] : []);
+  if (json) {
+    writeRawOutput(JSON.stringify(details.map((run) => run.result ?? stub(run)), null, 2));
+  } else {
+    const compare = details.length > 1 && results.length === details.length ? await request(apiUrl, token, "GET", `${EVALS}/compare?batch=${encodeURIComponent(started.batchId)}`) : null;
+    if (results.length)
+      writeRawOutput(
+        [...results.map(renderAgentEvalRun), ...compare ? [renderAgentEvalCompare(compare)] : []].join("\n").trimEnd()
+      );
+    for (const run of details.filter((run2) => !run2.result || run2.status !== "completed")) console.error(describeRun(run));
+  }
+  return exitCodeFor(details);
+}
+async function getRun(argv, target2, apiUrl, token) {
+  const json = hasFlag(argv, "--json");
+  const caseKey = getScalarArgValue(argv, "--case");
+  const transcript = hasFlag(argv, "--transcript");
+  if (transcript && !caseKey) throw new Error("--transcript needs --case <key>: transcripts are large, so the CLI prints one case at a time.");
+  const run = await request(apiUrl, token, "GET", runPath(target2, transcript));
+  if (!run.result) {
+    writeRawOutput(json ? JSON.stringify(stub(run), null, 2) : describeRun(run));
+    return;
+  }
+  const result = {
+    ...run.result,
+    cases: run.result.cases.filter((c) => caseKey === void 0 || c.key === caseKey).map((c) => ({
+      ...c,
+      attempts: c.attempts.map((attempt) => ({
+        ...attempt,
+        transcript: transcript ? run.attempts.find((a) => a.case.key === c.key && a.attemptIndex === attempt.index)?.transcript ?? attempt.transcript : attempt.transcript
+      }))
+    }))
+  };
+  if (caseKey && result.cases.length === 0) throw new Error(`Run ${target2} has no case "${caseKey}".`);
+  if (json) {
+    writeRawOutput(JSON.stringify(result, null, 2));
+    return;
+  }
+  const lines = [renderAgentEvalRun(result).trimEnd()];
+  if (transcript) {
+    for (const attempt of result.cases.flatMap((c) => c.attempts)) {
+      lines.push("", `Transcript, attempt ${attempt.index}:`);
+      lines.push(
+        ...attempt.transcript?.length ? attempt.transcript.map((entry) => `${entry.role}: ${entry.text}`) : ["(none: this eval is not multi-turn, or the attempt did not run)"]
+      );
+    }
+  }
+  writeRawOutput(lines.join("\n"));
+}
+async function listRuns(argv, name, apiUrl, token) {
+  const limit = intFlag(argv, "--limit");
+  const page = await request(
+    apiUrl,
+    token,
+    "GET",
+    `${EVALS}/${encodeURIComponent(name)}/runs${limit !== void 0 ? `?limit=${limit}` : ""}`
+  );
+  writeRawOutput(
+    hasFlag(argv, "--json") ? JSON.stringify(page, null, 2) : page.items.map((run) => `${run.id}  ${run.modelAlias}  ${run.status}  ${run.verdict ?? "-"}  ${run.commitSha.slice(0, 7)}  batch ${run.batchId}`).join("\n") || "No runs found."
+  );
+}
+async function compareRuns(argv, ids, apiUrl, token) {
+  const batch = getScalarArgValue(argv, "--batch");
+  if (Boolean(batch) === ids.length > 0) throw new Error("Use either one or more run ids or --batch <id>.");
+  const query = batch ? `batch=${encodeURIComponent(batch)}` : `runs=${ids.map(encodeURIComponent).join(",")}`;
+  const comparison = await request(apiUrl, token, "GET", `${EVALS}/compare?${query}`);
+  writeRawOutput(
+    hasFlag(argv, "--json") ? JSON.stringify(comparison.runs, null, 2) : renderAgentEvalCompare(comparison).trimEnd()
+  );
+}
+
+// src/eval-template.ts
+import { readFile as readFile2 } from "fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "path";
+import { parse } from "yaml";
+var RUNBOOK_TODO_MARKER = "<!-- cofactor:todo -->";
+var RUNBOOK_SECTIONS = [
+  {
+    heading: "What this eval proves",
+    prompt: "Describe the agent's job in one paragraph and the capabilities it backs."
+  },
+  {
+    heading: "Anatomy of a case",
+    prompt: "List the files in $EVAL_INPUT, what the expected value holds, and what a good case teaches."
+  },
+  {
+    heading: "Finding real data",
+    prompt: "Say where in production the inputs live: exact cofactor commands, queries and ids, and how to pick varied cases."
+  },
+  {
+    heading: "Labeling",
+    prompt: "Say how the expected value is decided and what must corroborate it; never the agent's own output alone."
+  },
+  {
+    heading: "Sanitizing",
+    prompt: "Say what to strip or replace before upload, the check to run, and the neutral names to use (Acme, example.com)."
+  },
+  {
+    heading: "Synthesizing scenarios",
+    prompt: "Say how to make a case when no real one exists (mutate a frozen input) and how to mark it synthetic in --source."
+  },
+  {
+    heading: "Adding the case",
+    prompt: "Give the exact eval add-case command shape for this eval."
+  },
+  { heading: "Known traps", prompt: "Record what has gone wrong before." }
+];
+var REQUIRED_SECTIONS = ["Finding real data", "Sanitizing"];
+var EVAL_GITIGNORE = "*.tar.gz\n_staging/\n";
+function initRunbook(name) {
+  return [
+    `# ${name} runbook`,
+    "",
+    ...RUNBOOK_SECTIONS.flatMap(({ heading, prompt }) => [`## ${heading}`, "", RUNBOOK_TODO_MARKER, `<!-- ${prompt} -->`, ""])
+  ].join("\n");
+}
+function initTemplate(opts) {
+  const capability = opts.capability ?? "CAP-REPLACE-ME";
+  const application = opts.application ?? "<application>";
+  return `# Agent eval: ${opts.name}
+# Push it with:
+#   cofactor eval push evals/${opts.name}.eval.yaml --application ${application} --org <org>
+# Add cases with:
+#   cofactor eval add-case ${opts.name} --key <key> --input <dir> --expected <file> --source "<provenance>" --org <org>
+# Run it with:
+#   cofactor eval run ${opts.name} --org <org>
+version: 1
+name: ${opts.name}
+description: Replace with one sentence on what this agent must get right.
+
+# Capabilities in the application's product map that this eval proves (CAP- codes).
+proves:
+  capabilities:
+    - ${capability}
+
+# Runbook for whoever adds cases; path relative to this file. Stored with the eval.
+runbook: ${opts.name}.runbook.md
+
+# single-turn: \`run\` is executed once per attempt. multi-turn: \`run\` is the agent process,
+# speaking JSONL on stdin/stdout, and a \`simulator\` model is required (cases then need a conversation).
+mode: single-turn
+
+# Shell command run at the repository root of the evaluated commit. It reads its case from
+# $EVAL_INPUT and writes its answer to $EVAL_OUTPUT/answer.* (answer.json, answer.txt, ...).
+# Model calls go to the injected ANTHROPIC_BASE_URL / OPENAI_BASE_URL; never use a real key.
+run: ./scripts/run-agent.sh
+
+input:
+  description: What the files in a case's input directory contain.
+  # Paths every case bundle must contain (files or directories); checked before upload.
+  require:
+    - input.json
+
+agent:
+  models:
+    # alias: provider (anthropic | azure) and model id. Aliases are what --model takes.
+    sonnet:
+      provider: anthropic
+      model: claude-sonnet-4-6
+  default: sonnet
+
+graders:
+  # A script grader runs after the agent exits with EVAL_INPUT, EVAL_OUTPUT and EVAL_EXPECTED
+  # (the case's expected value as a JSON file). Its last stdout line is
+  # {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader error.
+  - id: answer-matches-expected
+    kind: script
+    command: ./scripts/grade-answer.sh
+
+  # An ai-judge scores a rubric against the answer and the case's expected value.
+  # The judge model is pinned. scale is binary or 1-5; a 1-5 scale requires passAt.
+  - id: answer-quality
+    kind: ai-judge
+    rubric: The answer states the same root cause as the expected value and cites evidence from the input.
+    scale: 1-5
+    passAt: 4
+    judge:
+      provider: anthropic
+      model: claude-opus-4-8
+
+# Each case runs \`repeats\` times and passes when at least \`passAtLeast\` attempts pass.
+repeats: 3
+passAtLeast: 2
+
+# The eval passes when this share of cases pass.
+passBar: 0.5
+
+# Time limit per attempt, in seconds (30 to 1700).
+attemptSeconds: 600
+`;
+}
+function formatIssues(error) {
+  return error.issues.map((issue) => {
+    const path = issue.path.map(String).join(".");
+    return `${path || "(template)"}: ${issue.message}`;
+  });
+}
+var TemplateError = class extends Error {
+  constructor(heading, lines) {
+    super(`${heading}
+${lines.map((line) => `  ${line}`).join("\n")}`);
+  }
+};
+function sectionBodies(runbook) {
+  const sections = /* @__PURE__ */ new Map();
+  let current = null;
+  for (const line of runbook.split(/\r?\n/)) {
+    const heading = line.match(/^##\s+(.+?)\s*$/);
+    if (heading) {
+      current = heading[1];
+      sections.set(current, "");
+    } else if (current !== null) {
+      sections.set(current, `${sections.get(current)}${line}
+`);
+    }
+  }
+  return sections;
+}
+function checkRunbook(runbook) {
+  const sections = sectionBodies(runbook);
+  const errors = [];
+  const warnings = [];
+  const todo = [...sections].filter(([, body]) => body.split("\n").some((line) => line.trim() === RUNBOOK_TODO_MARKER));
+  if (todo.length) {
+    errors.push(
+      `Runbook still has ${RUNBOOK_TODO_MARKER} in: ${todo.map(([heading]) => `"## ${heading}"`).join(", ")}. Write each section, then delete its marker line.`
+    );
+  }
+  for (const { heading } of RUNBOOK_SECTIONS) {
+    const body = sections.get(heading);
+    const empty = body === void 0 || body.replace(/<!--[\s\S]*?-->/g, "").trim() === "";
+    if (body === void 0) warnings.push(`Runbook has no "## ${heading}" section.`);
+    if (REQUIRED_SECTIONS.includes(heading) && empty) {
+      errors.push(`Runbook section "## ${heading}" is empty; write it before pushing.`);
+    }
+  }
+  return { errors, warnings };
+}
+async function readPushPayload(file) {
+  let source;
+  try {
+    source = await readFile2(file, "utf8");
+  } catch {
+    throw new Error(`Cannot read ${file}.`);
+  }
+  let raw;
+  try {
+    raw = parse(source);
+  } catch (error) {
+    throw new Error(`${file} is not valid YAML: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const parsed = AgentEvalTemplateSchema.safeParse(raw);
+  if (!parsed.success) throw new TemplateError(`${file} is not a valid eval template:`, formatIssues(parsed.error));
+  const runbookPath = resolve(dirname(file), parsed.data.runbook);
+  const inside = relative(dirname(resolve(file)), runbookPath);
+  if (isAbsolute(parsed.data.runbook) || inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside))
+    throw new Error(`The template's runbook (${parsed.data.runbook}) must be a path inside the template's directory.`);
+  let runbook;
+  try {
+    runbook = await readFile2(runbookPath, "utf8");
+  } catch {
+    throw new Error(`Cannot read the runbook ${runbookPath} (the template's runbook: ${parsed.data.runbook}).`);
+  }
+  const { errors, warnings } = checkRunbook(runbook);
+  if (errors.length) throw new TemplateError(`${runbookPath} cannot be pushed:`, errors);
+  return { definition: parsed.data, runbook, warnings };
+}
+
+// src/eval.ts
+var USAGE = {
+  init: "init <name> [--application <app>] [--capability CAP-\u2026]",
+  push: "push [<file>] --application <app>",
+  show: "show <name>",
+  runbook: "runbook <name>",
+  list: "list",
+  cases: "cases <name>",
+  "rm-case": "rm-case <name> <key>",
+  "add-case": 'add-case <name> --key <k> --input <dir|file.tar.gz> --expected <file|-> --source "<provenance>" [--title <t>] [--conversation <file.json>]',
+  run: "run <name> [--model <alias>]... [--environment <env>] [--at <sha>] [--case <key>]... [--repeats N] [--parallel N] [--no-wait]",
+  get: "get <run-id> [--case <key>] [--transcript]",
+  runs: "runs <name> [--limit N]",
+  compare: "compare (<run-id>... | --batch <id>)"
+};
+var VALUE_FLAGS = [
+  ...GLOBAL_VALUE_FLAGS,
+  "--application",
+  "--capability",
+  "--key",
+  "--input",
+  "--expected",
+  "--source",
+  "--title",
+  "--conversation",
+  "--model",
+  "--environment",
+  "--at",
+  "--case",
+  "--repeats",
+  "--parallel",
+  "--limit",
+  "--batch"
+];
+var POSITIONALS = {
+  show: 1,
+  runbook: 1,
+  cases: 1,
+  "rm-case": 2,
+  "add-case": 1,
+  run: 1,
+  get: 1,
+  runs: 1
+};
+var WRITES = ["push", "add-case", "rm-case", "run"];
+var DEFAULT_TEMPLATE_DIR = "evals";
+function printHelp(command) {
+  const usage = Object.hasOwn(USAGE, command ?? "") ? USAGE[command] : void 0;
+  console.log(
+    [
+      `Usage: cofactor eval ${usage ?? "<sub-command>"} [--json]`,
+      "",
+      ...Object.values(USAGE).map((line) => `  cofactor eval ${line}`),
+      "",
+      "An eval runs an agent command against a set of cases at a commit, grades each attempt and reports",
+      "pass rate, cost and time per task. The template and runbook live in your repo under evals/.",
+      "",
+      "init: writes evals/<name>.eval.yaml (a commented template with a script and an ai-judge grader),",
+      "  evals/<name>.runbook.md and evals/.gitignore (*.tar.gz, _staging/). Refuses to overwrite.",
+      "push: validates the template locally, reads the runbook it names (relative to the YAML) and stores both;",
+      "  re-pushing is idempotent. Defaults to evals/*.eval.yaml when exactly one exists. Needs --application.",
+      '  Warns on a missing runbook heading; errors when "## Finding real data" or "## Sanitizing" is empty, or when',
+      '  any section still has the "<!-- cofactor:todo -->" line that init writes (delete it once the section is written).',
+      "show/list/cases/runbook: read the stored eval, its cases (key, sha256, size, source) and runbook.",
+      "add-case: --input is a directory (tarred deterministically: sorted entries, fixed mtime, uid/gid 0) or a .tar.gz.",
+      "  The sha256 shown is that of the uncompressed tar, so the same files give the same sha256 on any runtime",
+      "  and dedup is by tar content; gzip is transport only.",
+      "  The template's input.require paths are checked first.",
+      "  --expected is a JSON file, or - for stdin. --source says where the data came from and what corroborates the",
+      "  label. Never include customer data in a case. --conversation <file.json> is required for a multi-turn eval.",
+      "rm-case: deletes a case that no run has used.",
+      "run: starts one run per --model (alias from the template; none means its default) and waits, printing one",
+      "  section per model and then the comparison grid. --case, --repeats and a changed pass bar make a partial run",
+      "  that never counts for coverage or proofs. Polls every 5 seconds for at most 8.5 hours; a timeout leaves the",
+      "  runs readable with get; the batch and run ids are printed to stderr before waiting. --no-wait prints the",
+      "  ids and exits 0. --json prints an array with one entry per run, in order: the run's result, or",
+      "  {id, status, verdict, error} for a run with no result.",
+      "get: a run's result. --case <key> narrows to one case; --case <key> --transcript adds a multi-turn attempt's",
+      "  conversation. runs: recent runs for an eval, newest first (--limit N).",
+      "compare: the grid for run ids or a --batch; --json prints the array of runs.",
+      "",
+      "Common options:",
+      "  --org <name|id>    Logged-in organization; required explicitly for push, add-case, rm-case and run",
+      "  --env <env>        API environment (prod, dev, local)",
+      "  --api-url <url>    API URL override",
+      "  --token <key>      API token override",
+      "  --json             Raw JSON on every sub-command except init",
+      "  --help, -h         Show help without authentication",
+      "",
+      "Requires the sandbox-v2 organization flag. API-key scopes:",
+      "  run: sandbox:create + sandbox:exec, named explicitly (sandbox:* does not grant sandbox:exec).",
+      "  push, add-case, rm-case: sandbox:edit. show, list, cases, runbook, get, runs, compare: sandbox:read.",
+      "",
+      "Exit codes (run):",
+      "  0  every run's verdict is pass",
+      `  3  at least one run's verdict is fail (the agent missed the pass bar; perf also exits 3 for "slower")`,
+      "  1  a failed or cancelled run, a polling timeout, or any error",
+      "Other sub-commands exit 0 on success, 1 on error."
+    ].join("\n")
+  );
+}
+function required(argv, flag) {
+  const value = getScalarArgValue(argv, flag);
+  if (!value) throw new Error(`${flag} is required.`);
+  return value;
+}
+function expectedArg(argv) {
+  const i = argv.findIndex((arg) => arg === "--expected" || arg.startsWith("--expected="));
+  const value = i < 0 ? void 0 : argv[i].startsWith("--expected=") ? argv[i].slice("--expected=".length) : argv[i + 1];
+  if (!value || value.startsWith("--")) throw new Error("--expected is required: a JSON file, or - for stdin.");
+  return value;
+}
+async function readJson(path, label) {
+  let text;
+  try {
+    text = path === "-" ? await readStdin() : await readFile3(path, "utf8");
+  } catch (error) {
+    throw new Error(`Cannot read ${label}: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  try {
+    return JSON.parse(text);
+  } catch (error) {
+    throw new Error(`${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+function target(ctx, command) {
+  const value = ctx.positionals[0];
+  if (!value) throw new Error(`Usage: cofactor eval ${USAGE[command]}`);
+  return value;
+}
+async function init(argv) {
+  const name = getPositionalArg(argv, VALUE_FLAGS);
+  if (!name) throw new Error(`Usage: cofactor eval ${USAGE.init}`);
+  if (!/^[a-z0-9-]{2,40}$/.test(name) || name === "runs" || name === "compare")
+    throw new Error('The eval name must be 2 to 40 lowercase letters, digits or dashes (and not "runs" or "compare").');
+  const capability = getScalarArgValue(argv, "--capability");
+  if (capability !== void 0 && !/^CAP-[A-Z0-9]+-\d{3,}$/.test(capability))
+    throw new Error("--capability must be a product-map code such as CAP-CMM-001.");
+  const files = [
+    { path: join2(DEFAULT_TEMPLATE_DIR, `${name}.eval.yaml`), text: initTemplate({ name, application: getScalarArgValue(argv, "--application"), capability }) },
+    { path: join2(DEFAULT_TEMPLATE_DIR, `${name}.runbook.md`), text: initRunbook(name) }
+  ];
+  const gitignore = join2(DEFAULT_TEMPLATE_DIR, ".gitignore");
+  const exists = async (path) => readFile3(path).then(() => true, () => false);
+  for (const file of files) {
+    if (await exists(file.path)) throw new Error(`${file.path} already exists; not overwriting it.`);
+  }
+  await mkdir(DEFAULT_TEMPLATE_DIR, { recursive: true });
+  const written = files.map((file) => file.path);
+  for (const file of files) await writeFile(file.path, file.text, { flag: "wx" });
+  if (!await exists(gitignore)) {
+    await writeFile(gitignore, EVAL_GITIGNORE, { flag: "wx" });
+    written.push(gitignore);
+  }
+  writeRawOutput(
+    hasFlag(argv, "--json") ? JSON.stringify({ written }, null, 2) : `${written.map((path) => `Wrote ${path}`).join("\n")}
+Edit the run command and graders, write each runbook section and delete its <!-- cofactor:todo --> line, then: cofactor eval push ${files[0].path} --application <app> --org <org>`
+  );
+}
+async function defaultTemplateFile() {
+  const found = (await readdir2(DEFAULT_TEMPLATE_DIR).catch(() => [])).filter((f) => f.endsWith(".eval.yaml"));
+  if (found.length !== 1)
+    throw new Error(
+      found.length === 0 ? `No ${DEFAULT_TEMPLATE_DIR}/*.eval.yaml found; pass the template file.` : `Several templates in ${DEFAULT_TEMPLATE_DIR}/ (${found.join(", ")}); pass the one to push.`
+    );
+  return join2(DEFAULT_TEMPLATE_DIR, found[0]);
+}
+async function push({ argv, apiUrl, token, json }) {
+  const file = getPositionalArg(argv, VALUE_FLAGS) ?? await defaultTemplateFile();
+  const payload = await readPushPayload(file);
+  for (const warning of payload.warnings) console.error(`Warning: ${warning}`);
+  const applicationId = await resolveApplicationId(argv, apiUrl, token);
+  const pushed = await request(apiUrl, token, "PUT", EVALS, {
+    applicationId,
+    definition: payload.definition,
+    runbook: payload.runbook
+  });
+  const detail = await getEval(apiUrl, token, pushed.name);
+  if (json) {
+    writeRawOutput(JSON.stringify({ ...detail, warnings: payload.warnings }, null, 2));
+    return;
+  }
+  writeRawOutput(
+    [
+      `Pushed ${detail.name} (${detail.id})`,
+      `Cases: ${detail.cases.length}`,
+      `Capabilities: ${detail.capabilities.map((c) => c.code).join(", ") || "none"}`
+    ].join("\n")
+  );
+}
+function describeEval(detail) {
+  const d = detail.definition;
+  return [
+    `${detail.name}  ${detail.id}`,
+    d.description,
+    `Application: ${detail.applicationId}`,
+    `Mode: ${d.mode}`,
+    `Run: ${d.run.split("\n")[0]}${d.run.includes("\n") ? " \u2026" : ""}`,
+    `Models: ${Object.entries(d.agent.models).map(([alias, m]) => `${alias}${alias === d.agent.default ? "*" : ""} (${m.model})`).join(", ")}`,
+    `Graders: ${d.graders.map((g) => `${g.id} [${g.kind}]`).join(", ")}`,
+    `Repeats: ${d.repeats}, pass at least ${d.passAtLeast}, bar ${Math.round(d.passBar * 100)}%, ${d.attemptSeconds}s per attempt`,
+    `Capabilities: ${detail.capabilities.map((c) => `${c.code} ${c.title}`).join("; ") || "none"}`,
+    `Cases: ${detail.cases.length}`
+  ].join("\n");
+}
+async function listEvals({ apiUrl, token, json }) {
+  const evals = await requestAll(apiUrl, token, EVALS);
+  writeRawOutput(
+    json ? JSON.stringify(evals, null, 2) : evals.map((e) => `${e.name}  ${e.id}  ${e.caseCount} cases  ${e.latestVerdict ?? "-"}  ${e.description}`).join("\n") || "No evals found."
+  );
+}
+async function showEval(ctx) {
+  const detail = await getEval(ctx.apiUrl, ctx.token, target(ctx, "show"));
+  writeRawOutput(ctx.json ? JSON.stringify(detail, null, 2) : describeEval(detail));
+}
+async function listCases(ctx) {
+  const { cases } = await getEval(ctx.apiUrl, ctx.token, target(ctx, "cases"));
+  writeRawOutput(
+    ctx.json ? JSON.stringify(cases, null, 2) : cases.map((c) => `${c.key}  ${c.bundleSha256.slice(0, 12)}  ${c.bundleBytes} bytes  ${c.title}  [${c.source}]`).join("\n") || "No cases."
+  );
+}
+async function showRunbook(ctx) {
+  const name = target(ctx, "runbook");
+  const text = await requestText(ctx.apiUrl, ctx.token, `${EVALS}/${encodeURIComponent(name)}/runbook`);
+  writeRawOutput(ctx.json ? JSON.stringify({ name, runbook: text }, null, 2) : text.trimEnd());
+}
+async function removeCase(ctx) {
+  const name = target(ctx, "rm-case");
+  const key = ctx.positionals[1];
+  if (!key) throw new Error(`Usage: cofactor eval ${USAGE["rm-case"]}`);
+  const evaluation = await getEval(ctx.apiUrl, ctx.token, name);
+  await request(ctx.apiUrl, ctx.token, "DELETE", `${EVALS}/${encodeURIComponent(evaluation.id)}/cases/${encodeURIComponent(key)}`);
+  writeRawOutput(ctx.json ? JSON.stringify({ eval: name, key, deleted: true }) : `Removed case ${key} from ${name}`);
+}
+async function addCase(ctx) {
+  const { argv, apiUrl, token, json } = ctx;
+  const name = target(ctx, "add-case");
+  const key = required(argv, "--key");
+  const input = required(argv, "--input");
+  const expectedSource = expectedArg(argv);
+  const source = required(argv, "--source");
+  const conversationFile = getScalarArgValue(argv, "--conversation");
+  if (!/^[a-z0-9][a-z0-9-]{0,99}$/.test(key))
+    throw new Error("--key must be lowercase letters, digits and dashes, starting with a letter or digit (at most 100 characters).");
+  const expected = await readJson(expectedSource, expectedSource === "-" ? "the expected value on stdin" : `--expected ${expectedSource}`);
+  let conversation;
+  if (conversationFile) {
+    const parsed = EvalConversationSchema.safeParse(await readJson(conversationFile, `--conversation ${conversationFile}`));
+    if (!parsed.success)
+      throw new TemplateError(`${conversationFile} is not a valid conversation:`, formatIssues(parsed.error));
+    conversation = parsed.data;
+  }
+  const evaluation = await getEval(apiUrl, token, name);
+  const multiTurn = evaluation.definition.mode === "multi-turn";
+  if (multiTurn && !conversation) throw new Error(`${name} is multi-turn: pass --conversation <file.json>.`);
+  if (!multiTurn && conversation) throw new Error(`${name} is single-turn: --conversation is only for multi-turn evals.`);
+  const bundle = await loadBundle(input);
+  const missing = missingRequiredPaths(bundle.entries, evaluation.definition.input.require);
+  if (missing.length)
+    throw new Error(`The bundle is missing required input path(s): ${missing.join(", ")} (the template's input.require).`);
+  const evalPath = `${EVALS}/${encodeURIComponent(evaluation.id)}`;
+  const upload = await request(apiUrl, token, "POST", `${evalPath}/uploads`, {
+    sha256: bundle.sha256,
+    gzipSha256: bundle.gzipSha256,
+    bytes: bundle.bytes
+  });
+  const deduplicated = upload.status === "exists";
+  if (upload.status === "upload") {
+    const response = await fetch(upload.url, { method: "PUT", headers: upload.headers, body: new Uint8Array(bundle.archive) });
+    if (!response.ok)
+      throw new Error(`Uploading the bundle failed (${response.status}): ${(await response.text()).slice(0, 300)}`);
+  }
+  await request(apiUrl, token, "POST", `${evalPath}/cases`, {
+    key,
+    title: getScalarArgValue(argv, "--title") ?? key,
+    sha256: bundle.sha256,
+    expected,
+    source,
+    ...conversation ? { conversation } : {}
+  });
+  const result = { eval: name, key, sha256: bundle.sha256, bytes: bundle.bytes, deduplicated };
+  writeRawOutput(
+    json ? JSON.stringify(result, null, 2) : `Added case ${key} to ${name}
+sha256 ${bundle.sha256} (${bundle.bytes} bytes local gzip), ${deduplicated ? "deduplicated: the bundle already existed" : "uploaded"}`
+  );
+}
+var HANDLERS = {
+  push,
+  show: showEval,
+  runbook: showRunbook,
+  list: listEvals,
+  cases: listCases,
+  "rm-case": removeCase,
+  "add-case": addCase,
+  run: (ctx) => runEval(ctx.argv, target(ctx, "run"), ctx.apiUrl, ctx.token),
+  get: (ctx) => getRun(ctx.argv, target(ctx, "get"), ctx.apiUrl, ctx.token),
+  runs: (ctx) => listRuns(ctx.argv, target(ctx, "runs"), ctx.apiUrl, ctx.token),
+  compare: (ctx) => compareRuns(ctx.argv, ctx.positionals, ctx.apiUrl, ctx.token)
+};
+async function runEvalCommand(argv) {
+  const [command, ...rest] = argv;
+  if (!command || command === "help" || hasFlag(argv, "--help", "-h")) {
+    printHelp(command === "help" ? rest[0] : command);
+    return;
+  }
+  let code = 0;
+  try {
+    if (!Object.hasOwn(USAGE, command)) throw new Error(`Unknown eval command: ${command}`);
+    const cmd = command;
+    if (WRITES.includes(cmd)) required(rest, "--org");
+    for (const flag of VALUE_FLAGS.filter((flag2) => flag2 !== "--expected")) {
+      if (rest.includes(flag) && getScalarArgValue(rest, flag) === void 0) throw new Error(`${flag} requires a value.`);
+    }
+    const positionals = getPositionalArgs(rest, VALUE_FLAGS);
+    if (positionals.length < (POSITIONALS[cmd] ?? 0)) throw new Error(`Usage: cofactor eval ${USAGE[cmd]}`);
+    if (cmd === "init") {
+      await init(rest);
+    } else {
+      const { apiUrl, token } = await resolveConfig(rest);
+      const ctx = { argv: rest, positionals, apiUrl, token, json: hasFlag(rest, "--json") };
+      code = await HANDLERS[cmd](ctx) ?? 0;
+    }
+  } catch (error) {
+    console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+    code = 1;
+  }
+  if (code !== 0) process.exit(code);
+}
+export {
+  runEvalCommand
+};
+//# sourceMappingURL=eval-SO5TPD37.js.map