@garuda-agent/garuda 0.18.0 → 0.18.2

package/dist/chunk-A5DLAXB4.js

@@ -0,0 +1,1780 @@
+#!/usr/bin/env node
+import {
+  AutoApprover
+} from "./chunk-NXJ4XLB5.js";
+import {
+  DEFAULT_MAX_STEPS,
+  Runtime
+} from "./chunk-STVFWSIL.js";
+import {
+  FileSessionStore,
+  isCostIncomplete
+} from "./chunk-NLS7GMF7.js";
+import {
+  totalTokens
+} from "./chunk-7KBOIPUT.js";
+import {
+  LINK_CANDIDATES,
+  cleanArgs
+} from "./chunk-WK5NE6RY.js";
+import {
+  MANIFEST,
+  TEST_PATH,
+  tail
+} from "./chunk-AOGFYUKG.js";
+import {
+  git,
+  shellWord
+} from "./chunk-QXTHEFKP.js";
+import {
+  DEFAULT_ENV_ALLOWLIST,
+  HostExecutor,
+  denyAround,
+  parseSettings
+} from "./chunk-LMKGSYJC.js";
+import {
+  pathMatches
+} from "./chunk-3NJNVZTK.js";
+
+// src/evals/runner.ts
+import { constants as fsConstants, mkdirSync as mkdirSync3, realpathSync as realpathSync2 } from "fs";
+import { copyFile, mkdtemp as mkdtemp3, readFile as readFile4, rm as rm4, writeFile as writeFile4 } from "fs/promises";
+import { homedir, tmpdir as tmpdir4 } from "os";
+import { join as join7 } from "path";
+
+// src/evals/agents/claudeCode.ts
+import { mkdtemp, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+var MAX_OUTPUT_BYTES = 5e6;
+function claudeCommand(agent, prompt, maxTurns) {
+  return [
+    quote(agent.command),
+    "-p",
+    quote(prompt),
+    "--output-format",
+    "json",
+    "--model",
+    quote(agent.model),
+    "--max-turns",
+    String(maxTurns),
+    "--permission-mode",
+    "bypassPermissions"
+  ].join(" ");
+}
+function parseClaudeResult(stdout) {
+  const line = stdout.trim().split("\n").reverse().find((l) => l.trim().startsWith("{"));
+  let data;
+  try {
+    data = JSON.parse(line ?? "");
+  } catch {
+    return {
+      stopReason: "error",
+      steps: 0,
+      tokens: 0,
+      cacheReadTokens: 0,
+      costUsd: void 0,
+      reason: `Claude Code gave no JSON result: ${stdout.trim().slice(0, 300) || "no output"}`
+    };
+  }
+  const usage = data.usage ?? {};
+  const n = (v) => typeof v === "number" && Number.isFinite(v) ? v : 0;
+  const cacheRead = n(usage.cache_read_input_tokens);
+  const tokens = n(usage.input_tokens) + n(usage.output_tokens) + cacheRead + n(usage.cache_creation_input_tokens);
+  const costUsd = typeof data.total_cost_usd === "number" ? data.total_cost_usd : void 0;
+  const steps = n(data.num_turns);
+  const subtype = typeof data.subtype === "string" ? data.subtype : "";
+  const base = { steps, tokens, cacheReadTokens: cacheRead, costUsd };
+  if (subtype === "error_max_turns") return { ...base, stopReason: "max_steps" };
+  if (subtype === "success" && data.is_error !== true) return { ...base, stopReason: "done" };
+  const text = typeof data.result === "string" ? data.result : subtype || "unknown error";
+  return { ...base, stopReason: "error", reason: `Claude Code failed: ${text.slice(0, 300)}` };
+}
+async function runClaudeCode(executor, policy, agent, prompt, maxTurns, signal) {
+  const config = await mkdtemp(join(tmpdir(), "garuda-claude-config-"));
+  try {
+    await writeFile(
+      join(config, ".claude.json"),
+      `${JSON.stringify({ hasCompletedOnboarding: true })}
+`
+    );
+    const result = await executor.run(
+      claudeCommand(agent, prompt, maxTurns),
+      { ...policy, maxOutputBytes: MAX_OUTPUT_BYTES },
+      {
+        signal,
+        env: {
+          ...agent.env,
+          CLAUDE_CONFIG_DIR: config,
+          CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
+          DISABLE_AUTOUPDATER: "1"
+        }
+      }
+    );
+    const outcome = result.timedOut || result.aborted ? {
+      stopReason: "timeout",
+      steps: 0,
+      tokens: 0,
+      cacheReadTokens: 0,
+      costUsd: void 0,
+      reason: "Claude Code took longer than the task's time limit."
+    } : parseClaudeResult(result.stdout.text);
+    if (outcome.stopReason === "error" && result.stderr.text.trim() !== "" && outcome.reason !== void 0) {
+      outcome.reason += `
+${result.stderr.text.trim().split("\n").slice(-3).join("\n")}`;
+    }
+    return { result, outcome };
+  } finally {
+    await rm(config, { recursive: true, force: true });
+  }
+}
+function quote(text) {
+  return `'${text.replace(/'/g, "'\\''")}'`;
+}
+
+// src/evals/changes.ts
+import { createHash } from "crypto";
+import { lstat, readdir, readFile, readlink } from "fs/promises";
+import { join as join2 } from "path";
+var IGNORED_DIRS = /* @__PURE__ */ new Set([
+  ".git",
+  ".garuda",
+  // Claude Code's own project folder (the T11b baseline), as .garuda is Garuda's.
+  ".claude",
+  "node_modules",
+  "target",
+  "build",
+  "dist",
+  "__pycache__",
+  ".pytest_cache"
+]);
+var IGNORED_FILE = /\.(class|pyc)$/;
+var MAX_TRACKED_FILES = 5e4;
+async function treeState(root) {
+  const out = /* @__PURE__ */ new Map();
+  const walk = async (dir, prefix) => {
+    let names;
+    try {
+      names = await readdir(dir);
+    } catch {
+      return;
+    }
+    for (const name of names.sort()) {
+      if (out.size >= MAX_TRACKED_FILES) return;
+      const path = prefix === "" ? name : `${prefix}/${name}`;
+      const full = join2(dir, name);
+      const info = await lstat(full).catch(() => void 0);
+      if (info === void 0) continue;
+      if (info.isDirectory()) {
+        if (!IGNORED_DIRS.has(name)) await walk(full, path);
+      } else if (info.isSymbolicLink()) {
+        out.set(path, `link:${await readlink(full).catch(() => "")}`);
+      } else if (info.isFile() && !IGNORED_FILE.test(name)) {
+        const hash = createHash("sha256");
+        hash.update(await readFile(full).catch(() => Buffer.alloc(0)));
+        out.set(path, hash.digest("hex"));
+      }
+    }
+  };
+  await walk(root, "");
+  return out;
+}
+function changedPaths(before, after) {
+  const out = /* @__PURE__ */ new Set();
+  for (const [path, hash] of after) if (before.get(path) !== hash) out.add(path);
+  for (const path of before.keys()) if (!after.has(path)) out.add(path);
+  return [...out].sort();
+}
+function unexpectedPaths(changed, expected, allowChanges = []) {
+  const known = new Set(expected);
+  return changed.filter(
+    (path) => !known.has(path) && !allowChanges.some((glob) => pathMatches(glob, path, false))
+  );
+}
+
+// src/evals/compare.ts
+import { readFile as readFile2, stat } from "fs/promises";
+import { join as join3 } from "path";
+
+// src/evals/stats.ts
+var BOOTSTRAP_DRAWS = 1e4;
+var DEFAULT_SEED = 1;
+function seeded(seed) {
+  let a = seed >>> 0;
+  return () => {
+    a = a + 1831565813 >>> 0;
+    let t = a;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function shuffled(items, seed) {
+  const out = [...items];
+  const random = seeded(seed);
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(random() * (i + 1));
+    [out[i], out[j]] = [out[j], out[i]];
+  }
+  return out;
+}
+function byTask(results) {
+  const map = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    let t = map.get(r.id);
+    if (t === void 0) {
+      t = { id: r.id, runs: 0, passes: 0, errors: 0, results: [] };
+      map.set(r.id, t);
+    }
+    t.runs++;
+    if (r.passed) t.passes++;
+    if (r.stopReason === "error") t.errors++;
+    t.results.push(r);
+  }
+  return [...map.values()];
+}
+function passInterval(results, options = {}) {
+  const tasks = byTask(results);
+  const runs = tasks.reduce((s, t) => s + t.runs, 0);
+  if (tasks.length === 0 || runs === 0) return void 0;
+  const seed = options.seed ?? DEFAULT_SEED;
+  const draws = options.draws ?? BOOTSTRAP_DRAWS;
+  const level = options.level ?? 0.95;
+  const random = seeded(seed);
+  const rates = new Float64Array(draws);
+  for (let d = 0; d < draws; d++) {
+    let passes = 0;
+    let n = 0;
+    for (let i = 0; i < tasks.length; i++) {
+      const t = tasks[Math.floor(random() * tasks.length)];
+      passes += t.passes;
+      n += t.runs;
+    }
+    rates[d] = passes / n;
+  }
+  rates.sort();
+  const tail2 = (1 - level) / 2;
+  return {
+    rate: tasks.reduce((s, t) => s + t.passes, 0) / runs,
+    low: quantile(rates, tail2),
+    high: quantile(rates, 1 - tail2),
+    level,
+    draws,
+    seed,
+    tasks: tasks.length
+  };
+}
+function consistency(results) {
+  const repeated = byTask(results).filter((t) => t.runs >= 2);
+  return {
+    repeated: repeated.length,
+    allPassed: repeated.filter((t) => t.passes === t.runs).map((t) => t.id),
+    nonePassed: repeated.filter((t) => t.passes === 0 && t.errors < t.runs).map((t) => t.id),
+    onlyErrors: repeated.filter((t) => t.errors === t.runs).map((t) => t.id)
+  };
+}
+function spread(results) {
+  const scored = results.filter((r) => r.stopReason !== "error");
+  if (scored.length === 0) return void 0;
+  const of = (f) => {
+    const sorted = Float64Array.from(scored.map(f)).sort();
+    return { median: quantile(sorted, 0.5), p95: quantile(sorted, 0.95) };
+  };
+  return {
+    runs: scored.length,
+    steps: of((r) => r.steps),
+    tokens: of((r) => r.tokens),
+    durationMs: of((r) => r.durationMs)
+  };
+}
+function stepsSd(runs) {
+  const steps = runs.filter((r) => r.stopReason !== "error").map((r) => r.steps);
+  if (steps.length < 2) return void 0;
+  const mean = steps.reduce((s, x) => s + x, 0) / steps.length;
+  return Math.sqrt(steps.reduce((s, x) => s + (x - mean) ** 2, 0) / (steps.length - 1));
+}
+function costPerPass(results) {
+  if (results.some((r) => r.costUsd === void 0)) return { kind: "unknown" };
+  const totalUsd = results.reduce((s, r) => s + (r.costUsd ?? 0), 0);
+  const passes = results.filter((r) => r.passed).length;
+  if (passes === 0) return { kind: "none", totalUsd };
+  return {
+    kind: "value",
+    usd: totalUsd / passes,
+    minimum: results.some((r) => r.costIncomplete === true),
+    totalUsd,
+    passes
+  };
+}
+function unexpectedChanges(results) {
+  const measured = results.filter((r) => r.unexpected !== void 0);
+  return {
+    runs: measured.filter((r) => (r.unexpected?.length ?? 0) > 0).length,
+    files: measured.reduce((s, r) => s + (r.unexpected?.length ?? 0), 0),
+    measured: measured.length
+  };
+}
+function quantile(sorted, q) {
+  if (sorted.length === 0) return Number.NaN;
+  const pos = (sorted.length - 1) * q;
+  const lo = Math.floor(pos);
+  const hi = Math.ceil(pos);
+  const a = sorted[lo];
+  const b = sorted[hi];
+  return a + (b - a) * (pos - lo);
+}
+function mcpUse(results) {
+  const measured = results.filter((r) => r.mcpCalls !== void 0);
+  if (measured.length === 0) return void 0;
+  return {
+    calls: measured.reduce((sum, r) => sum + (r.mcpCalls ?? 0), 0),
+    runs: measured.filter((r) => (r.mcpCalls ?? 0) > 0).length,
+    measured: measured.length
+  };
+}
+function mcpUseText(use) {
+  return `${use.calls} in ${use.runs} of ${use.measured} runs${use.runs === 0 ? " (the agent never used the server)" : ""}`;
+}
+
+// src/evals/compare.ts
+async function loadReport(path) {
+  const file = (await stat(path)).isDirectory() ? join3(path, "report.json") : path;
+  const data = JSON.parse(await readFile2(file, "utf8"));
+  return parseReport(data, file);
+}
+function parseReport(data, path) {
+  if (!Array.isArray(data.results)) throw new Error(`${path} is not an eval report (no results).`);
+  const schema = typeof data.schema === "number" ? data.schema : 1;
+  const tasks = schema >= 2 && data.tasks !== null && typeof data.tasks === "object" ? data.tasks : void 0;
+  return {
+    path,
+    schema,
+    model: typeof data.spec === "string" ? data.spec : typeof data.model === "string" ? data.model : void 0,
+    seed: typeof data.seed === "number" ? data.seed : void 0,
+    tasks,
+    results: data.results,
+    mcp: mcpServerOf(data.mcp)
+  };
+}
+function mcpServerOf(value) {
+  if (value === null || typeof value !== "object") return void 0;
+  const server = value.server;
+  return typeof server === "string" ? server : void 0;
+}
+function compareReports(a, b, options = {}) {
+  const problems = [];
+  const notes = [];
+  const idsA = [...new Set(a.results.map((r) => r.id))].sort();
+  const idsB = [...new Set(b.results.map((r) => r.id))].sort();
+  if (idsA.join("\n") !== idsB.join("\n")) {
+    const onlyA = idsA.filter((id) => !idsB.includes(id));
+    const onlyB = idsB.filter((id) => !idsA.includes(id));
+    problems.push(
+      `The reports ran different tasks${onlyA.length > 0 ? `; only in A: ${onlyA.join(", ")}` : ""}${onlyB.length > 0 ? `; only in B: ${onlyB.join(", ")}` : ""}.`
+    );
+  }
+  if (a.tasks !== void 0 && b.tasks !== void 0) {
+    const changed = idsA.filter((id) => idsB.includes(id) && a.tasks?.[id] !== b.tasks?.[id]);
+    if (changed.length > 0) {
+      problems.push(
+        `These tasks changed between the runs (their hashes differ): ${changed.join(", ")}. Run both arms on the same tasks.`
+      );
+    }
+  } else {
+    const old = [a, b].filter((r) => r.tasks === void 0).map((r) => r.path);
+    notes.push(
+      `${old.join(" and ")} ${old.length === 1 ? "has" : "have"} no task hashes (a report before 0.17): Garuda cannot check that the tasks were the same.`
+    );
+  }
+  if (problems.length > 0) return { problems };
+  if (idsA.length === 0) return { problems: ["The reports have no runs."] };
+  const tasksA = new Map(byTask(a.results).map((t) => [t.id, t]));
+  const tasksB = new Map(byTask(b.results).map((t) => [t.id, t]));
+  const rows = idsA.map((id) => {
+    const ta = tasksA.get(id);
+    const tb = tasksB.get(id);
+    const arm = (t) => {
+      const results = t?.results ?? [];
+      return {
+        passes: t?.passes ?? 0,
+        runs: t?.runs ?? 0,
+        costUsd: results.some((r) => r.costUsd === void 0) ? void 0 : results.reduce((s, r) => s + (r.costUsd ?? 0), 0),
+        minimum: results.some((r) => r.costIncomplete === true)
+      };
+    };
+    return { id, a: arm(ta), b: arm(tb) };
+  });
+  const rate = (side, list) => list.reduce((s, r) => s + r[side].passes, 0) / list.reduce((s, r) => s + r[side].runs, 0);
+  const rateA = rate("a", rows);
+  const rateB = rate("b", rows);
+  const seed = options.seed ?? DEFAULT_SEED;
+  const draws = options.draws ?? BOOTSTRAP_DRAWS;
+  const level = options.level ?? 0.95;
+  const random = seeded(seed);
+  const diffs = new Float64Array(draws);
+  for (let d = 0; d < draws; d++) {
+    let pa = 0;
+    let na = 0;
+    let pb = 0;
+    let nb = 0;
+    for (let i = 0; i < rows.length; i++) {
+      const r = rows[Math.floor(random() * rows.length)];
+      pa += r.a.passes;
+      na += r.a.runs;
+      pb += r.b.passes;
+      nb += r.b.runs;
+    }
+    diffs[d] = pb / nb - pa / na;
+  }
+  diffs.sort();
+  const tail2 = (1 - level) / 2;
+  const low = quantile(diffs, tail2);
+  const high = quantile(diffs, 1 - tail2);
+  const verdict = low > 0 ? "B passes more" : high < 0 ? "B passes less" : "no clear difference";
+  if (low === high) {
+    notes.push(
+      "Every draw gave the same pass-rate difference, so the interval shows no uncertainty: more tasks, or tasks that sometimes fail, are needed to measure it."
+    );
+  }
+  if (rows.length < 10) notes.push(roughNote(rows.length));
+  if (a.seed !== void 0 && b.seed !== void 0 && a.seed !== b.seed) {
+    notes.push(
+      `The arms ran with different seeds (${a.seed} and ${b.seed}): their run orders differ.`
+    );
+  }
+  if (a.mcp !== b.mcp) {
+    notes.push(
+      `The arms differ in the MCP server: A ${a.mcp ?? "none"}, B ${b.mcp ?? "none"}. For an MCP A/B, keep every other option the same.`
+    );
+  }
+  const cost = costRatio(rows, seed, draws, level);
+  if (typeof cost === "string") notes.push(cost);
+  else if (cost.skipped > draws * 0.05) {
+    notes.push(
+      `The cost interval skipped ${cost.skipped} of ${draws} draws (an arm had no pass in them).`
+    );
+  }
+  if (rows.some((r) => r.a.minimum || r.b.minimum)) {
+    notes.push("A run's cost is a minimum (no reported usage): the cost ratio is approximate.");
+  }
+  const runsA = rows.map((r) => r.a.runs);
+  const runsB = rows.map((r) => r.b.runs);
+  if (Math.min(...runsA, ...runsB) < 3) {
+    notes.push(
+      "Some tasks have fewer than 3 runs in an arm: repeat each task 3 times or more (--repeat 3)."
+    );
+  }
+  return {
+    comparison: {
+      rows,
+      rateA,
+      rateB,
+      difference: { value: rateB - rateA, low, high, level, draws, seed },
+      verdict,
+      ...typeof cost === "string" ? {} : { costRatio: cost },
+      notes
+    }
+  };
+}
+function costRatio(rows, seed, draws, level) {
+  if (rows.some((r) => r.a.costUsd === void 0 || r.b.costUsd === void 0)) {
+    return "A run's cost is unknown: no cost comparison.";
+  }
+  const per = (list, side) => {
+    const passes = list.reduce((s, r) => s + r[side].passes, 0);
+    const cost = list.reduce((s, r) => s + (r[side].costUsd ?? 0), 0);
+    return passes === 0 ? void 0 : cost / passes;
+  };
+  const a = per(rows, "a");
+  const b = per(rows, "b");
+  if (a === void 0 || b === void 0 || a === 0) {
+    return "An arm has no pass (or no cost): no cost ratio.";
+  }
+  const random = seeded(seed);
+  const ratios = [];
+  const drawn = new Array(rows.length);
+  for (let d = 0; d < draws; d++) {
+    for (let i = 0; i < rows.length; i++) {
+      drawn[i] = rows[Math.floor(random() * rows.length)];
+    }
+    const ca = per(drawn, "a");
+    const cb = per(drawn, "b");
+    if (ca !== void 0 && cb !== void 0 && ca > 0) ratios.push(cb / ca);
+  }
+  const sorted = Float64Array.from(ratios).sort();
+  const tail2 = (1 - level) / 2;
+  const low = quantile(sorted, tail2);
+  const high = quantile(sorted, 1 - tail2);
+  return {
+    value: b / a,
+    low,
+    high,
+    draws,
+    skipped: draws - ratios.length,
+    verdict: low > 1 ? "B costs more per accepted change" : high < 1 ? "B costs less per accepted change" : "no clear difference"
+  };
+}
+function roughNote(tasks) {
+  return `Only ${tasks} task${tasks === 1 ? "" : "s"}: with fewer than 10 tasks the interval is a rough guide.`;
+}
+var pct = (x) => `${Math.round(x * 100)}%`;
+var signed = (x) => {
+  const points = Math.round(Math.abs(x) * 100);
+  return points === 0 ? "0 pts" : `${x > 0 ? "+" : "\u2212"}${points} pts`;
+};
+function formatComparison(a, b, c) {
+  const width = Math.max(4, ...c.rows.map((r) => r.id.length));
+  const lines2 = [
+    `A: ${a.path}${a.model === void 0 ? "" : ` (${a.model})`}`,
+    `B: ${b.path}${b.model === void 0 ? "" : ` (${b.model})`}`,
+    "",
+    `${"Task".padEnd(width)}  A        B`,
+    ...c.rows.map(
+      (r) => `${r.id.padEnd(width)}  ${`${r.a.passes}/${r.a.runs}`.padEnd(7)}  ${r.b.passes}/${r.b.runs}`
+    ),
+    "",
+    `Pass rate over all attempts: A ${pct(c.rateA)}, B ${pct(c.rateB)}; B \u2212 A ${signed(c.difference.value)} (95% paired interval ${signed(c.difference.low)} to ${signed(c.difference.high)}; ${c.difference.draws} draws, seed ${c.difference.seed})`,
+    `Verdict: ${c.verdict}${c.verdict === "no clear difference" ? " (the interval contains 0)" : ""}.`
+  ];
+  for (const [name, report] of [
+    ["A", a],
+    ["B", b]
+  ]) {
+    const cost = costPerPass(report.results);
+    const u = unexpectedChanges(report.results);
+    const s = spread(report.results);
+    const mcp = mcpUse(report.results);
+    lines2.push(
+      `${name}: cost per accepted change ${cost.kind === "unknown" ? "unknown" : cost.kind === "none" ? "no passes" : `${cost.minimum ? "\u2265 " : ""}$${cost.usd.toFixed(4)}`} \xB7 unexpected changes ${u.measured === 0 ? "not measured" : `${u.runs} of ${u.measured} runs`} \xB7 median time ${s === void 0 ? "\u2013" : `${Math.round(s.durationMs.median / 1e3)} s`}${mcp === void 0 ? "" : ` \xB7 MCP tool calls ${mcpUseText(mcp)}`}`
+    );
+  }
+  if (c.costRatio !== void 0) {
+    const x = (v) => `\xD7${v.toFixed(2)}`;
+    lines2.push(
+      `Cost per accepted change: B/A ${x(c.costRatio.value)} (95% paired interval ${x(c.costRatio.low)} to ${x(c.costRatio.high)}). Verdict: ${c.costRatio.verdict}${c.costRatio.verdict === "no clear difference" ? " (the interval contains \xD71.00)" : ""}.`
+    );
+  }
+  for (const note of c.notes) lines2.push(`Note: ${note}`);
+  return lines2.join("\n");
+}
+
+// src/evals/files.ts
+import { mkdirSync } from "fs";
+import { writeFile as writeFile2 } from "fs/promises";
+import { dirname, join as join4 } from "path";
+async function writeFiles(root, files) {
+  for (const [path, content] of Object.entries(files)) {
+    const full = join4(root, path);
+    mkdirSync(dirname(full), { recursive: true });
+    await writeFile2(full, content);
+  }
+}
+
+// src/evals/projects.ts
+var MAVEN_POM = `<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>dev.garuda.eval</groupId>
+  <artifactId>eval</artifactId>
+  <version>1.0.0</version>
+
+  <properties>
+    <maven.compiler.release>17</maven.compiler.release>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.11.4</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <version>3.3.1</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.13.0</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.5.2</version>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+`;
+var MAVEN_CHECK = "test ! -e .mvn && mvn -B -q -o test";
+var PYPROJECT = `[project]
+name = "evalpkg"
+version = "0.1.0"
+requires-python = ">=3.9"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+`;
+var PYTEST_CHECK = "python3 -m pytest -q -c pyproject.toml --noconftest -p no:cacheprovider";
+function isProtectedPath(path) {
+  return /^(test|tests|src\/test)\//.test(path) || path === "pom.xml" || path === "pyproject.toml";
+}
+var lines = (...l) => `${l.join("\n")}
+`;
+
+// src/evals/repoTasks.ts
+import { mkdirSync as mkdirSync2, mkdtempSync, realpathSync } from "fs";
+import { mkdir, readFile as readFile3, rm as rm3, symlink, writeFile as writeFile3 } from "fs/promises";
+import { tmpdir as tmpdir3 } from "os";
+import { dirname as dirname2, join as join6, sep } from "path";
+import { z } from "zod";
+
+// src/evals/toolchains.ts
+import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join5 } from "path";
+var TOOLCHAIN_ENV = ["JAVA_HOME", "GRADLE_USER_HOME", "VIRTUAL_ENV"];
+var TOOLCHAINS = {
+  maven: {
+    id: "maven",
+    title: "Maven with JUnit 5 in ~/.m2",
+    files: {
+      "pom.xml": MAVEN_POM,
+      "src/main/java/probe/Probe.java": "package probe;\n\npublic final class Probe {\n  public static int one() {\n    return 1;\n  }\n}\n",
+      "src/test/java/probe/ProbeTest.java": "package probe;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport org.junit.jupiter.api.Test;\n\nclass ProbeTest {\n  @Test\n  void one() {\n    assertEquals(1, Probe.one());\n  }\n}\n"
+    },
+    probe: "mvn -B -q -o test",
+    prepare: "mvn -B -q test",
+    hint: "The Java tasks need a JDK (17 or later), Maven, and JUnit 5 in ~/.m2. Run `garuda eval --prepare java` once: it downloads the plugins and JUnit with Maven (network)."
+  },
+  pytest: {
+    id: "pytest",
+    title: "Python 3 with pytest",
+    files: {
+      "pyproject.toml": PYPROJECT,
+      "src/probe/__init__.py": "def one():\n    return 1\n",
+      "tests/test_probe.py": "from probe import one\n\n\ndef test_one():\n    assert one() == 1\n"
+    },
+    probe: "python3 -m pytest -q -p no:cacheprovider",
+    hint: "The Python tasks need python3 with pytest 7 or later. Install it, for example with `python3 -m pip install --user pytest`."
+  }
+};
+var PROBE_TIMEOUT_MS = 12e4;
+var PREPARE_TIMEOUT_MS = 10 * 6e4;
+async function checkToolchains(ids) {
+  return Promise.all(
+    [...new Set(ids)].map(async (id) => {
+      const toolchain = TOOLCHAINS[id];
+      const r = await inScratch(toolchain, toolchain.probe, PROBE_TIMEOUT_MS);
+      return { id, ...r };
+    })
+  );
+}
+async function prepareToolchain(id) {
+  const toolchain = TOOLCHAINS[id];
+  if (toolchain.prepare === void 0) return { ok: false, output: toolchain.hint };
+  return inScratch(toolchain, toolchain.prepare, PREPARE_TIMEOUT_MS);
+}
+function toolchainId(name) {
+  const aliases = {
+    java: "maven",
+    maven: "maven",
+    python: "pytest",
+    pytest: "pytest"
+  };
+  return aliases[name.toLowerCase()];
+}
+async function inScratch(toolchain, command, timeoutMs) {
+  const root = await mkdtemp2(join5(tmpdir2(), `garuda-toolchain-${toolchain.id}-`));
+  try {
+    await writeFiles(root, toolchain.files);
+    const r = await new HostExecutor().run(command, {
+      root,
+      sandbox: false,
+      writePaths: [root],
+      denyWritePaths: [],
+      denyReadPaths: [],
+      network: true,
+      envAllowlist: [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV],
+      timeoutMs,
+      maxOutputBytes: 4e3
+    });
+    const output = `${r.stdout.text}${r.stderr.text}`.trim();
+    return { ok: r.exitCode === 0 && !r.timedOut, output };
+  } catch (error) {
+    return { ok: false, output: error.message };
+  } finally {
+    await rm2(root, { recursive: true, force: true });
+  }
+}
+
+// src/evals/repoTasks.ts
+var REPO_SUITE_FILE = join6(".garuda", "evals", "repo-suite.json");
+var DEFAULT_COMMITS = 200;
+var DEFAULT_MAX_TASKS = 30;
+var MAX_TASK_FILES = 10;
+var BUILD_TEST_TIMEOUT_MS = 5 * 6e4;
+var testFile = z.object({ path: z.string(), content: z.string().nullable() });
+var taskSchema = z.object({
+  id: z.string(),
+  sha: z.string(),
+  base: z.string(),
+  subject: z.string(),
+  prompt: z.string(),
+  tests: z.array(testFile),
+  /** The check: the test command for this task's test files. Absent: the suite's test command. */
+  check: z.string().optional()
+});
+var suiteSchema = z.object({
+  version: z.literal(1),
+  root: z.string(),
+  head: z.string(),
+  createdAt: z.string(),
+  testCommand: z.string(),
+  tasks: z.array(taskSchema)
+});
+var DOC_PATH = /\.(md|mdx|rst|txt|adoc)$|(^|\/)docs?\//i;
+var PACKAGE_JSON = /(^|\/)package\.json$/;
+function skipReason(commit) {
+  const { files } = commit;
+  if (files.length === 0) return "no files";
+  const code = files.filter((f) => !DOC_PATH.test(f.path));
+  if (code.length > MAX_TASK_FILES) return `more than ${MAX_TASK_FILES} files`;
+  if (files.some((f) => f.added === void 0)) return "a binary file";
+  if (files.some((f) => MANIFEST.test(f.path) && !PACKAGE_JSON.test(f.path))) {
+    return "dependencies changed";
+  }
+  const tests = code.filter((f) => TEST_PATH.test(f.path));
+  if (tests.length === 0) return "no test change";
+  if (tests.length === code.length) return "only tests changed";
+  return void 0;
+}
+function dependencyChange(diff) {
+  for (const line of diff.split("\n")) {
+    if (!/^[+-]/.test(line) || /^(\+\+\+|---)/.test(line)) continue;
+    if (/[dD]ependencies"/.test(line)) return true;
+    const entry = /^[+-]\s*"([^"]+)":\s*"[~^<>=]*\d/.exec(line);
+    if (entry !== null && entry[1] !== "version") return true;
+  }
+  return false;
+}
+function scopedTestCommand(command, files) {
+  const list = files.map(shellWord).join(" ");
+  if (command.includes("{files}")) return command.replaceAll("{files}", list);
+  if (files.length === 0) return command;
+  const c = command.trim();
+  if (/^npm (run )?test$/.test(c)) return `${c} -- ${list}`;
+  if (/^(pnpm|yarn)( run)? test$/.test(c) || /^node --test$/.test(c) || /^(npx |pnpm exec )?(vitest|jest)( run)?$/.test(c) || /-m pytest( -q)?$|^pytest( -q)?$/.test(c)) {
+    return `${c} ${list}`;
+  }
+  return c;
+}
+function repoPrompt(commit, tests) {
+  return [
+    commit.subject.trim(),
+    ...commit.body.trim() === "" ? [] : ["", commit.body.trim()],
+    "",
+    `The tests for this change are already in the project: ${tests.join(", ")}. Make the change so that they pass. Do not change these test files.`
+  ].join("\n");
+}
+async function recentCommits(executor, root, options = {}) {
+  const log = await git(executor, root, [
+    "log",
+    "--no-merges",
+    "--format=%H%x00%s%x00%b%x1e",
+    `-n${options.commits ?? DEFAULT_COMMITS}`,
+    ...options.since === void 0 ? [] : [`--since=${options.since}`],
+    "HEAD"
+  ]);
+  const out = [];
+  for (const entry of log.stdout.split("")) {
+    const [sha = "", subject = "", body = ""] = entry.replace(/^\n/, "").split("\0");
+    if (!/^[0-9a-f]{40,64}$/.test(sha)) continue;
+    const stat2 = await git(executor, root, ["show", "--no-renames", "--numstat", "--format=", sha]);
+    const files = stat2.stdout.split("\n").map((line) => /^(\d+|-)\t(\d+|-)\t(.+)$/.exec(line)).filter((m) => m !== null).map(
+      (m) => m[1] === "-" ? { path: m[3] } : { path: m[3], added: Number(m[1]), removed: Number(m[2]) }
+    );
+    out.push({ sha, subject, body, files });
+  }
+  return out;
+}
+async function taskWorktree(executor, root, commit) {
+  const dir = realpathSync(mkdtempSync(join6(tmpdir3(), "garuda-repo-task-")));
+  await git(executor, root, ["worktree", "add", "--detach", "--quiet", dir, commit]);
+  for (const name of LINK_CANDIDATES) {
+    try {
+      await symlink(join6(root, name), join6(dir, name), "dir");
+    } catch {
+    }
+  }
+  return dir;
+}
+async function isolatedTaskCopy(executor, root, commit) {
+  const dir = realpathSync(mkdtempSync(join6(tmpdir3(), "garuda-repo-task-")));
+  const indexDir = mkdtempSync(join6(tmpdir3(), "garuda-repo-index-"));
+  try {
+    const env = { GIT_INDEX_FILE: join6(indexDir, "index") };
+    await git(executor, root, ["read-tree", commit], { env });
+    await git(executor, root, ["checkout-index", "--all", "--force", `--prefix=${dir}/`], { env });
+  } finally {
+    await rm3(indexDir, { recursive: true, force: true });
+  }
+  await git(executor, dir, ["init", "--quiet"]);
+  await git(executor, dir, ["add", "--all"]);
+  await git(executor, dir, [
+    "-c",
+    "user.name=Garuda eval",
+    "-c",
+    "user.email=eval@garuda.invalid",
+    "commit",
+    "--quiet",
+    "--no-verify",
+    "--message",
+    "The task's start"
+  ]);
+  for (const name of LINK_CANDIDATES) {
+    try {
+      await symlink(join6(root, name), join6(dir, name), "dir");
+    } catch {
+    }
+  }
+  return dir;
+}
+function repoTaskReadDenials(root, home) {
+  const links = LINK_CANDIDATES.map((name) => join6(root, name));
+  const inHome = root === home || root.startsWith(`${home}${sep}`);
+  return [
+    ...inHome ? denyAround(home, [...links, join6(home, "Library")], true) : [],
+    ...inHome ? [] : denyAround(root, links)
+  ];
+}
+var NODE_MODULES_CACHES = [".vite-temp", ".vite", ".cache"];
+function repoTaskWritePaths(root) {
+  let modules;
+  try {
+    modules = realpathSync(join6(root, "node_modules"));
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const name of NODE_MODULES_CACHES) {
+    const path = join6(modules, name);
+    try {
+      mkdirSync2(path, { recursive: true });
+      out.push(path);
+    } catch {
+    }
+  }
+  return out;
+}
+async function removeTaskWorktree(executor, root, dir) {
+  await git(executor, root, ["worktree", "remove", "--force", dir], { check: false });
+  await rm3(dir, { recursive: true, force: true });
+  await git(executor, root, ["worktree", "prune"], { check: false });
+}
+async function applyTests(dir, tests) {
+  for (const t of tests) {
+    const file = join6(dir, t.path);
+    if (t.content === null) await rm3(file, { force: true });
+    else {
+      await mkdir(dirname2(file), { recursive: true });
+      await writeFile3(file, t.content);
+    }
+  }
+}
+async function runTests(executor, dir, command) {
+  const result = await executor.run(command, {
+    root: dir,
+    sandbox: false,
+    writePaths: [dir],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV],
+    timeoutMs: BUILD_TEST_TIMEOUT_MS,
+    maxOutputBytes: 2e4
+  });
+  const output = `${result.stdout.text}${result.stderr.text}`;
+  return {
+    ok: result.exitCode === 0 && !result.timedOut,
+    tail: result.timedOut ? "(stopped: too long)" : tail(output, 15)
+  };
+}
+async function buildRepoSuite(executor, root, options) {
+  const say = options.onProgress ?? (() => {
+  });
+  const head = (await git(executor, root, ["rev-parse", "HEAD"])).stdout.trim();
+  const commits = await recentCommits(executor, root, {
+    ...options.commits === void 0 ? {} : { commits: options.commits },
+    ...options.since === void 0 ? {} : { since: options.since }
+  });
+  const skipped = {};
+  const skip = (why) => {
+    skipped[why] = (skipped[why] ?? 0) + 1;
+  };
+  const tasks = [];
+  let checked = 0;
+  let failure;
+  let evalFailure;
+  const max = options.maxTasks ?? DEFAULT_MAX_TASKS;
+  for (const commit of commits) {
+    if (tasks.length >= max) break;
+    const why = skipReason(commit);
+    if (why !== void 0) {
+      skip(why);
+      continue;
+    }
+    const parent = await git(
+      executor,
+      root,
+      ["rev-parse", "--verify", "--quiet", `${commit.sha}^`],
+      {
+        check: false
+      }
+    );
+    if (parent.exitCode !== 0) {
+      skip("the first commit (no parent)");
+      continue;
+    }
+    if (commit.files.some((f) => PACKAGE_JSON.test(f.path))) {
+      const diff = await git(
+        executor,
+        root,
+        ["show", "--format=", commit.sha, "--", "package.json"],
+        {
+          check: false
+        }
+      );
+      if (dependencyChange(diff.stdout)) {
+        skip("dependencies changed");
+        continue;
+      }
+    }
+    const base = parent.stdout.trim();
+    const testPaths = commit.files.filter((f) => TEST_PATH.test(f.path) && !DOC_PATH.test(f.path)).map((f) => f.path);
+    const tests = [];
+    for (const path of testPaths) {
+      const shown = await git(executor, root, ["show", `${commit.sha}:${path}`], { check: false });
+      tests.push({ path, content: shown.exitCode === 0 ? shown.stdout : null });
+    }
+    const runnable = tests.filter((t) => t.content !== null).map((t) => t.path);
+    if (runnable.length === 0) {
+      skip("only deletes tests");
+      continue;
+    }
+    const check = scopedTestCommand(options.testCommand, runnable);
+    checked++;
+    say(`checking ${commit.sha.slice(0, 7)} ${commit.subject}`);
+    const dir = await taskWorktree(executor, root, base);
+    try {
+      await applyTests(dir, tests);
+      if ((await runTests(executor, dir, check)).ok) {
+        skip("the tests already pass at the parent");
+        continue;
+      }
+      await git(executor, dir, ["checkout", "--quiet", "--", "."], { check: false });
+      await git(executor, dir, cleanArgs(), { check: false });
+      await git(executor, dir, ["checkout", "--quiet", "--detach", commit.sha]);
+      const atCommit = await runTests(executor, dir, check);
+      if (!atCommit.ok) {
+        skip("the tests fail at the commit");
+        failure ??= { sha: commit.sha, command: check, tail: atCommit.tail };
+        continue;
+      }
+    } finally {
+      await removeTaskWorktree(executor, root, dir);
+    }
+    if (options.evalCheck !== void 0) {
+      const inEval = await options.evalCheck(commit.sha, check);
+      if (!inEval.ok) {
+        skip("the tests fail as an eval runs them (a task copy, in the sandbox)");
+        evalFailure ??= { sha: commit.sha, command: check, tail: inEval.tail };
+        continue;
+      }
+    }
+    tasks.push({
+      id: `repo-${commit.sha.slice(0, 7)}`,
+      sha: commit.sha,
+      base,
+      subject: commit.subject,
+      prompt: repoPrompt(commit, testPaths),
+      tests,
+      check
+    });
+    say(`kept ${commit.sha.slice(0, 7)} (${tasks.length}/${max})`);
+  }
+  return {
+    suite: {
+      version: 1,
+      root,
+      head,
+      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+      testCommand: options.testCommand,
+      tasks
+    },
+    checked,
+    skipped,
+    ...failure === void 0 ? {} : { failure },
+    ...evalFailure === void 0 ? {} : { evalFailure }
+  };
+}
+async function saveRepoSuite(suite) {
+  const file = join6(suite.root, REPO_SUITE_FILE);
+  await mkdir(dirname2(file), { recursive: true });
+  await writeFile3(file, `${JSON.stringify(suite, null, 2)}
+`, { mode: 384 });
+  return file;
+}
+async function loadRepoSuite(root) {
+  let text;
+  try {
+    text = await readFile3(join6(root, REPO_SUITE_FILE), "utf8");
+  } catch {
+    return void 0;
+  }
+  const parsed = suiteSchema.safeParse(JSON.parse(text));
+  if (!parsed.success) throw new Error(`${REPO_SUITE_FILE}: ${z.prettifyError(parsed.error)}`);
+  return parsed.data;
+}
+function repoEvalTasks(suite) {
+  return suite.tasks.map((t) => ({
+    id: t.id,
+    title: t.subject,
+    prompt: t.prompt,
+    files: {},
+    check: t.check ?? suite.testCommand,
+    protect: t.tests.filter((f) => f.content !== null).map((f) => f.path),
+    solution: {},
+    repo: { root: suite.root, base: t.base, sha: t.sha, tests: t.tests }
+  }));
+}
+
+// src/evals/runner.ts
+var EVAL_DENY_RULES = [
+  "bash(rm -rf*)",
+  "bash(sudo*)",
+  "bash(git push*)",
+  "bash(curl*)",
+  "bash(wget*)"
+];
+var REPO_CHECK_TIMEOUT_MS = 5 * 6e4;
+var EVAL_TASK_TIMEOUT_MS = 10 * 6e4;
+var PROVIDER_STALL_MS = 12e4;
+var PROVIDER_STALLED = "The model's API stalled:";
+var MCP_PREPARE_TIMEOUT_MS = 12e4;
+var ModelWatch = class {
+  steps = 0;
+  /** Tool calls that started and have no result yet. */
+  running = 0;
+  /** The last response asked for tools that have not started yet. */
+  toolsNext = false;
+  lastAt = Date.now();
+  retried = false;
+  see(event, now = Date.now()) {
+    switch (event.type) {
+      case "model_retry":
+        this.retried = true;
+        return;
+      case "tool_call":
+        this.running++;
+        this.toolsNext = false;
+        break;
+      case "tool_result":
+        this.running = Math.max(0, this.running - 1);
+        break;
+      case "step_end":
+        this.steps = event.step;
+        this.toolsNext = event.response.content.some((b) => b.type === "tool_use");
+        this.retried = false;
+        break;
+    }
+    this.lastAt = now;
+  }
+  /** A model request is open: no tool runs or is about to. */
+  get waiting() {
+    return this.running === 0 && !this.toolsNext;
+  }
+  /** Why the stop is the provider's, or undefined: a tool was running, or the model was sending. */
+  stall(now, quietMs) {
+    if (!this.waiting) return void 0;
+    const quiet = Math.round((now - this.lastAt) / 1e3);
+    if (this.retried) return `its stream broke and was retried, and it sent nothing for ${quiet} s`;
+    if (now - this.lastAt >= quietMs) return `it sent nothing for ${quiet} s`;
+    return void 0;
+  }
+};
+function repoTaskSandbox(repoRoot, home) {
+  return {
+    denyRead: repoTaskReadDenials(repoRoot, home),
+    writePaths: repoTaskWritePaths(repoRoot)
+  };
+}
+async function checkAsEval(repoRoot, sha, check, options = {}) {
+  const host = new HostExecutor();
+  const root = await isolatedTaskCopy(host, repoRoot, sha);
+  try {
+    const runtime = await Runtime.create({
+      root,
+      modelId: "none",
+      model: async () => {
+        throw new Error("checkAsEval asks no model.");
+      },
+      approver: new AutoApprover("once"),
+      store: new FileSessionStore(root),
+      settings: parseSettings({
+        permissions: { deny: EVAL_DENY_RULES },
+        web: { enabled: false },
+        sandbox: repoTaskSandbox(repoRoot, options.home ?? homedir()),
+        ...options.executor === void 0 ? {} : { executor: options.executor }
+      }),
+      mcp: false,
+      hooks: false,
+      commands: false,
+      ...options.policy === void 0 ? {} : { policy: options.policy }
+    });
+    try {
+      const result = await runCheck(
+        root,
+        check,
+        REPO_CHECK_TIMEOUT_MS,
+        runtime.executor.isolation === "none" ? void 0 : { executor: runtime.executor, policy: runtime.execPolicy(REPO_CHECK_TIMEOUT_MS) }
+      );
+      return { ok: result.ok, tail: result.output.split("\n").slice(-15).join("\n") };
+    } finally {
+      await runtime.close();
+    }
+  } finally {
+    await rm4(root, { recursive: true, force: true });
+  }
+}
+async function runEvalTask(task, options) {
+  const started = Date.now();
+  const host = new HostExecutor();
+  let root;
+  if (task.repo === void 0) {
+    root = realpathSync2(await mkdtemp3(join7(tmpdir4(), `garuda-eval-${task.id}-`)));
+    await writeFiles(root, task.files);
+  } else {
+    root = await isolatedTaskCopy(host, task.repo.root, task.repo.base);
+    await applyTests(root, task.repo.tests);
+  }
+  if (options.format !== void 0) await formatProject(root, options.format.biome);
+  const protect = task.protect ?? Object.keys(task.files).filter(isProtectedPath);
+  const before = await snapshot(root, protect);
+  const settings = parseSettings({
+    // With an allowlist, a host outside it is denied: nobody answers questions in an eval.
+    permissions: {
+      deny: options.network === void 0 ? EVAL_DENY_RULES : [...EVAL_DENY_RULES, "network"]
+    },
+    ...options.network === void 0 ? {} : { network: { allow: options.network } },
+    // Evals must not depend on the internet.
+    web: { enabled: false },
+    // 0.18.1: a repo task's commands cannot read the project's own folder or the rest of the
+    // home folder: the checkout, and copies of it, hold the commit that the task asks for.
+    ...task.repo === void 0 ? {} : { sandbox: repoTaskSandbox(task.repo.root, options.home ?? homedir()) },
+    ...options.executor === void 0 ? {} : { executor: options.executor },
+    ...options.codeIndex === void 0 ? {} : { codeIndex: options.codeIndex },
+    ...options.maxSteps === void 0 ? {} : { limits: { maxSteps: options.maxSteps } },
+    ...options.subagents === void 0 ? {} : { subagents: { enabled: options.subagents } },
+    ...options.todo === void 0 ? {} : { todo: { enabled: options.todo } },
+    ...options.format === void 0 ? {} : {
+      formatters: {
+        enabled: options.format.on,
+        commands: {
+          biome: {
+            extensions: ["js", "jsx", "mjs", "cjs", "ts", "tsx", "json"],
+            command: [options.format.biome, "format", "--write", "$FILE"]
+          }
+        }
+      }
+    },
+    ...options.keepThinking === void 0 ? {} : { thinking: { keepBlocks: options.keepThinking } }
+  });
+  const store = new FileSessionStore(root);
+  const watch = new ModelWatch();
+  let mcpCalls = 0;
+  const tools = [];
+  const createRuntime = (prepare2) => Runtime.create({
+    root,
+    modelId: options.modelId,
+    model: async () => options.model(),
+    ...options.modelInfo === void 0 ? {} : { modelInfo: options.modelInfo },
+    ...options.maxTokens === void 0 ? {} : { maxTokens: options.maxTokens },
+    ...options.subagentModel === void 0 ? {} : { subagentModel: options.subagentModel },
+    approver: new AutoApprover("once"),
+    store,
+    settings,
+    // The user's own MCP servers must not change eval results; `--mcp` allows one by name.
+    mcp: options.mcp === void 0 || prepare2 ? false : { home: options.mcp.home, only: options.mcp.server },
+    hooks: false,
+    ...options.addendum === void 0 ? {} : { addendum: options.addendum },
+    ...options.policy === void 0 ? {} : { policy: options.policy },
+    commands: false,
+    // The list comes from the eval command line: the user typed it.
+    ...options.network === void 0 ? {} : { network: { approved: true } },
+    ...options.lsp === void 0 ? {} : { lsp: { enabled: options.lsp } },
+    onEvent: (event) => {
+      if (event.type === "tool_call") {
+        tools.push(event.call.name);
+        if (event.call.name.startsWith("mcp__")) mcpCalls++;
+      }
+      watch.see(event);
+      options.onEvent?.(task.id, event);
+    }
+  });
+  let prepareFailure;
+  const prepare = options.mcp?.prepare;
+  if (prepare !== void 0) {
+    const prep = await createRuntime(true);
+    try {
+      const prepared = await runCheck(
+        root,
+        prepare,
+        MCP_PREPARE_TIMEOUT_MS,
+        prep.executor.isolation === "none" ? void 0 : { executor: prep.executor, policy: prep.execPolicy(MCP_PREPARE_TIMEOUT_MS) }
+      );
+      if (!prepared.ok) prepareFailure = `${MCP_PREPARE_FAILED}
+${prepared.output}`;
+    } finally {
+      await prep.close();
+      prep.executor.shutdown();
+    }
+  }
+  const runtime = await createRuntime(false);
+  const result = {
+    id: task.id,
+    title: task.title,
+    passed: false,
+    stopReason: "error",
+    steps: 0,
+    tokens: 0,
+    costUsd: void 0,
+    durationMs: 0
+  };
+  if (prepareFailure !== void 0) result.reason = prepareFailure;
+  const start = await treeState(root);
+  const limitMs = options.taskTimeoutMs ?? EVAL_TASK_TIMEOUT_MS;
+  const timeout = AbortSignal.timeout(limitMs);
+  try {
+    if (result.reason !== void 0) {
+    } else if (options.agent !== void 0) {
+      await runtime.prepareCommands(timeout);
+      const { result: exec, outcome } = await runClaudeCode(
+        runtime.executor,
+        runtime.execPolicy(limitMs),
+        options.agent,
+        task.prompt,
+        options.maxSteps ?? DEFAULT_MAX_STEPS,
+        timeout
+      );
+      result.stopReason = outcome.stopReason;
+      result.steps = outcome.steps;
+      result.tokens = outcome.tokens;
+      result.cacheReadTokens = outcome.cacheReadTokens;
+      result.costUsd = outcome.costUsd;
+      if (outcome.reason !== void 0) result.reason = outcome.reason;
+      if (options.outDir !== void 0) {
+        mkdirSync3(options.outDir, { recursive: true });
+        result.sessionFile = await saveUnique(
+          options.outDir,
+          task.id,
+          ".claude.json",
+          (path) => writeFile4(path, exec.stdout.text, { flag: "wx" })
+        );
+      }
+    } else {
+      const run = await runtime.runTurn(task.prompt, timeout);
+      result.stopReason = run.stopReason;
+      result.steps = run.steps;
+    }
+  } catch (error) {
+    runtime.recordStop("error");
+    result.steps = watch.steps;
+    const stall = timeout.aborted ? watch.stall(Date.now(), options.providerStallMs ?? PROVIDER_STALL_MS) : void 0;
+    if (stall !== void 0) {
+      result.stopReason = "error";
+      result.reason = `${PROVIDER_STALLED} ${stall} when the task hit its ${limitMs / 6e4}-minute limit.`;
+    } else if (timeout.aborted) {
+      result.stopReason = "timeout";
+      result.reason = `The task took longer than ${limitMs / 6e4} minutes.`;
+    } else {
+      result.reason = `The run failed: ${error.message}`;
+    }
+  } finally {
+    if (tools.length > 0) result.tools = tools;
+    if (options.mcp !== void 0) {
+      result.mcpCalls = mcpCalls;
+      const status = runtime.mcpStatus().find((m) => m.name === options.mcp?.server);
+      if (result.reason === void 0 && status !== void 0 && status.state === "failed") {
+        result.stopReason = "error";
+        result.reason = `${MCP_NOT_STARTED} "${options.mcp.server}": ${status.message ?? "no reason given"}`;
+      }
+    }
+    await runtime.close();
+    runtime.executor.shutdown();
+  }
+  const session = runtime.session;
+  if (session !== void 0 && options.agent === void 0) {
+    result.tokens = totalTokens(session.usage);
+    if (session.unreportedTokens > 0) result.unreportedTokens = session.unreportedTokens;
+    result.costUsd = session.costUsd;
+    if (isCostIncomplete(session)) result.costIncomplete = true;
+    result.cacheReadTokens = session.usage.cacheReadTokens;
+    if (options.outDir !== void 0) {
+      mkdirSync3(options.outDir, { recursive: true });
+      const from = store.path(session.id);
+      result.sessionFile = await saveUnique(
+        options.outDir,
+        task.id,
+        ".jsonl",
+        (path) => copyFile(from, path, fsConstants.COPYFILE_EXCL)
+      );
+    }
+  }
+  result.changed = changedPaths(start, await treeState(root));
+  const expected = await expectedFiles(task, host);
+  if (expected !== void 0) {
+    result.unexpected = unexpectedPaths(result.changed, expected, task.allowChanges);
+  }
+  if (result.reason === void 0) {
+    const changed = await changedFiles(root, before);
+    const checkMs = task.repo === void 0 ? 12e4 : REPO_CHECK_TIMEOUT_MS;
+    const check = await runCheck(
+      root,
+      task.check,
+      checkMs,
+      runtime.executor.isolation === "none" ? void 0 : { executor: runtime.executor, policy: runtime.execPolicy(checkMs) }
+    );
+    if (changed.length > 0) result.reason = `${PROTECTED_FILES_CHANGED} ${changed.join(", ")}.`;
+    else if (!check.ok) result.reason = `The check failed:
+${check.output}`;
+    else result.passed = true;
+  }
+  result.durationMs = Date.now() - started;
+  if (!options.keep) {
+    await rm4(root, { recursive: true, force: true });
+  }
+  return result;
+}
+async function expectedFiles(task, host) {
+  if (task.repo === void 0) return Object.keys(task.solution);
+  if (task.repo.sha === void 0) return void 0;
+  const r = await git(
+    host,
+    task.repo.root,
+    ["diff", "--name-only", task.repo.base, task.repo.sha],
+    {
+      check: false
+    }
+  );
+  if (r.exitCode !== 0) return void 0;
+  const tests = new Set(task.repo.tests.map((t) => t.path));
+  return r.stdout.split("\n").map((line) => line.trim()).filter((line) => line !== "" && !tests.has(line));
+}
+function isAuthFailure(result) {
+  if (!isErrorRun(result) || result.reason === void 0) return false;
+  return /\b40[13]\b|authentication_error|permission_error|invalid x-api-key|invalid api key/i.test(
+    result.reason
+  );
+}
+function isConnectionFailure(result) {
+  if (!isErrorRun(result) || result.reason === void 0) return false;
+  return /ECONNREFUSED|ENOTFOUND|EAI_AGAIN|connection refused|self[- ]signed certificate|unable to get local issuer certificate|certificate has expired/i.test(
+    result.reason
+  );
+}
+async function saveUnique(dir, id, ext, write) {
+  for (let n = 1; ; n++) {
+    const path = join7(dir, `${id}${n === 1 ? "" : `-${n}`}${ext}`);
+    try {
+      await write(path);
+      return path;
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+    }
+  }
+}
+var PROTECTED_FILES_CHANGED = "The agent changed protected files:";
+var MCP_PREPARE_FAILED = "The MCP prepare command failed:";
+var MCP_NOT_STARTED = "The MCP server did not start:";
+function isMcpFailure(result) {
+  return result.reason !== void 0 && (result.reason.startsWith(MCP_PREPARE_FAILED) || result.reason.startsWith(MCP_NOT_STARTED));
+}
+function isProviderStall(result) {
+  return isErrorRun(result) && result.reason?.startsWith(PROVIDER_STALLED) === true;
+}
+function isSetupFailure(result) {
+  if (isProviderStall(result)) return false;
+  return isAuthFailure(result) || isConnectionFailure(result) || isMcpFailure(result);
+}
+var FAILURE_KINDS = [
+  "check",
+  "protected-files",
+  "step-limit",
+  "token-limit",
+  "repeated-calls",
+  "refusal",
+  "timeout",
+  "provider",
+  "error",
+  "setup"
+];
+function failureKind(result) {
+  if (result.passed) return void 0;
+  if (isErrorRun(result)) {
+    if (isProviderStall(result)) return "provider";
+    return isSetupFailure(result) ? "setup" : "error";
+  }
+  switch (result.stopReason) {
+    case "timeout":
+      return "timeout";
+    case "max_steps":
+      return "step-limit";
+    case "token_budget":
+    case "max_tokens":
+      return "token-limit";
+    case "repeated_calls":
+      return "repeated-calls";
+    case "refusal":
+      return "refusal";
+  }
+  if (result.reason?.startsWith(PROTECTED_FILES_CHANGED) === true) return "protected-files";
+  return "check";
+}
+function failureSummary(results) {
+  const runs = [];
+  for (const r of results) {
+    const kind = failureKind(r);
+    if (kind === void 0) continue;
+    runs.push({
+      id: r.id,
+      attempt: r.attempt ?? 1,
+      kind,
+      steps: r.steps,
+      tokens: r.tokens,
+      durationMs: r.durationMs
+    });
+  }
+  const byKind = {};
+  for (const kind of FAILURE_KINDS) {
+    const n = runs.filter((f) => f.kind === kind).length;
+    if (n > 0) byKind[kind] = n;
+  }
+  return { total: runs.length, byKind, runs };
+}
+function failureLine(results) {
+  const s = failureSummary(results);
+  if (s.total === 0) return void 0;
+  const parts = FAILURE_KINDS.flatMap((kind) => {
+    const runs = s.runs.filter((f) => f.kind === kind);
+    if (runs.length === 0) return [];
+    const named = runs.slice(0, 3).map((f) => `${f.id} run ${f.attempt}`);
+    if (runs.length > 3) named.push(`${runs.length - 3} more`);
+    return [`${kind} ${runs.length} (${named.join(", ")})`];
+  });
+  return `Failures by kind: ${parts.join(" \xB7 ")}`;
+}
+function withStalledRun(again, stalled) {
+  const unreported = (again.unreportedTokens ?? 0) + (stalled.unreportedTokens ?? 0);
+  return {
+    ...again,
+    tokens: again.tokens + stalled.tokens,
+    ...unreported > 0 ? { unreportedTokens: unreported } : {},
+    costUsd: again.costUsd === void 0 || stalled.costUsd === void 0 ? void 0 : again.costUsd + stalled.costUsd,
+    ...again.costIncomplete || stalled.costIncomplete ? { costIncomplete: true } : {},
+    providerStalls: (again.providerStalls ?? 0) + 1
+  };
+}
+async function runEvals(tasks, options, onResult, stop) {
+  const results = new Array(tasks.length);
+  const seen = /* @__PURE__ */ new Map();
+  const attempts = tasks.map((t) => {
+    const n = (seen.get(t.id) ?? 0) + 1;
+    seen.set(t.id, n);
+    return n;
+  });
+  let next = 0;
+  let stopped = false;
+  const worker = async () => {
+    for (let i = next++; i < tasks.length && !stopped; i = next++) {
+      let result = await runEvalTask(tasks[i], options);
+      if (isProviderStall(result)) {
+        options.onEvent?.(result.id, {
+          type: "notice",
+          text: `${result.reason ?? PROVIDER_STALLED} Running it again.`
+        });
+        result = withStalledRun(await runEvalTask(tasks[i], options), result);
+      }
+      result.attempt = attempts[i];
+      results[i] = result;
+      onResult?.(result);
+      if (stop?.(result)) stopped = true;
+    }
+  };
+  const width = Math.max(1, Math.min(options.parallel ?? 1, tasks.length));
+  await Promise.all(Array.from({ length: width }, worker));
+  return results.filter((r) => r !== void 0);
+}
+async function runCheck(root, command, timeoutMs = 12e4, sandboxed) {
+  const envAllowlist = [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV];
+  const r = sandboxed === void 0 ? await new HostExecutor().run(command, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist,
+    timeoutMs,
+    maxOutputBytes: 4e3
+  }) : await sandboxed.executor.run(command, {
+    ...sandboxed.policy,
+    envAllowlist: [.../* @__PURE__ */ new Set([...sandboxed.policy.envAllowlist, ...TOOLCHAIN_ENV])],
+    timeoutMs,
+    maxOutputBytes: 4e3
+  });
+  const text = `${r.stdout.text}${r.stderr.text}`.trim();
+  const output = text !== "" ? text : `(no output; ${r.timedOut ? "timed out" : `exit code ${r.exitCode}`})`;
+  return { ok: r.exitCode === 0 && !r.timedOut, output };
+}
+async function snapshot(root, paths) {
+  const out = /* @__PURE__ */ new Map();
+  for (const path of paths) out.set(path, await readFile4(join7(root, path), "utf8").catch(() => ""));
+  return out;
+}
+async function changedFiles(root, before) {
+  const after = await snapshot(root, [...before.keys()]);
+  return [...before.keys()].filter((path) => before.get(path) !== after.get(path));
+}
+function isErrorRun(result) {
+  return result.stopReason === "error";
+}
+function summarize(results) {
+  const attempted = results.length;
+  const errors = results.filter(isErrorRun).length;
+  const scored = attempted - errors;
+  const passed = results.filter((r) => r.passed).length;
+  return {
+    attempted,
+    errors,
+    scored,
+    passed,
+    ...scored === 0 ? {} : { scoredRate: passed / scored },
+    ...attempted === 0 ? {} : { attemptedRate: passed / attempted }
+  };
+}
+var percent = (rate) => rate === void 0 ? "\u2013" : `${Math.round(rate * 100)}%`;
+function formatReport(results, options = {}) {
+  const width = idWidth(results);
+  const rows = results.map(
+    (r) => [
+      r.passed ? "PASS" : isErrorRun(r) ? "ERR " : "FAIL",
+      r.id.padEnd(width),
+      `${String(r.steps).padStart(3)} steps`,
+      (r.unreportedTokens ?? 0) > 0 ? `tok ? (${r.unreportedTokens} reserved)` : `${(r.tokens / 1e3).toFixed(1).padStart(7)}k tok`,
+      r.costUsd === void 0 ? "   cost ?" : `${r.costIncomplete === true ? "\u2265" : ""}$${r.costUsd.toFixed(4).padStart(7)}`,
+      `${(r.durationMs / 1e3).toFixed(0).padStart(4)} s`,
+      r.stopReason
+    ].join("  ")
+  );
+  const summary = summarize(results);
+  const known = `$${results.reduce((s, r) => s + (r.costUsd ?? 0), 0).toFixed(4)}`;
+  const cost = !results.every((r) => r.costUsd !== void 0) ? "unknown" : results.some((r) => r.costIncomplete === true) ? `\u2265 ${known} (incomplete)` : known;
+  const steps = results.reduce((s, r) => s + r.steps, 0);
+  const counted = `${summary.passed}/${summary.scored} scored runs passed (${percent(summary.scoredRate)}) \xB7 ${summary.passed}/${summary.attempted} of all attempted runs (${percent(summary.attemptedRate)})` + (summary.errors > 0 ? ` \xB7 ${summary.errors} error run(s)` : "");
+  const tokens = results.reduce((s, r) => s + r.tokens, 0);
+  const cached = results.reduce((s, r) => s + (r.cacheReadTokens ?? 0), 0);
+  const share = results.some((r) => (r.unreportedTokens ?? 0) > 0) ? " \xB7 cache share unknown" : tokens === 0 ? "" : ` \xB7 ${Math.round(cached / tokens * 100)}% of tokens from the cache`;
+  const lines2 = [
+    ...rows,
+    "",
+    `${counted} \xB7 ${steps} steps \xB7 cost ${cost}${share}`,
+    ...evidenceLines(results, options.seed ?? DEFAULT_SEED)
+  ];
+  const ids = [...new Set(results.map((r) => r.id))];
+  if (ids.length < results.length) lines2.push("", "Mean per task:", ...meanRows(results, ids));
+  return lines2.join("\n");
+}
+function evidenceLines(results, seed) {
+  const out = [];
+  if (results.length > 0 && results.every(isErrorRun)) {
+    return [
+      "No scored run: every run failed with an error (the reasons are above), so this run gives no evidence about the agent."
+    ];
+  }
+  const interval = passInterval(results, { seed });
+  if (interval !== void 0 && interval.tasks >= 2) {
+    out.push(
+      `Pass rate over all attempts: ${percent(interval.rate)} (95% interval ${percent(interval.low)}\u2013${percent(interval.high)}; cluster bootstrap over ${interval.tasks} tasks, ${interval.draws} draws, seed ${interval.seed})`
+    );
+    if (interval.low === interval.high) {
+      out.push(
+        "  Every draw gave the same rate, so the interval shows no uncertainty: more tasks are needed to measure it."
+      );
+    }
+    if (interval.tasks < 10) out.push(`  ${roughNote(interval.tasks)}`);
+  }
+  const c = consistency(results);
+  if (c.repeated > 0) {
+    const none = c.nonePassed.length === 0 ? "" : ` (${c.nonePassed.join(", ")})`;
+    const errors = c.onlyErrors.length === 0 ? "" : `; ${c.onlyErrors.length} had only error runs (${c.onlyErrors.join(", ")})`;
+    out.push(
+      `Consistency: ${c.allPassed.length} of ${c.repeated} repeated tasks passed every run; ${c.nonePassed.length} passed none${none}${errors}`
+    );
+  }
+  const s = spread(results);
+  if (s !== void 0) {
+    const k = (n) => `${(n / 1e3).toFixed(1)}k`;
+    const sec = (ms) => `${Math.round(ms / 1e3)} s`;
+    out.push(
+      `Spread (${s.runs} scored runs): steps median ${s.steps.median.toFixed(1)}, p95 ${s.steps.p95.toFixed(1)} \xB7 tokens median ${k(s.tokens.median)}, p95 ${k(s.tokens.p95)} \xB7 time median ${sec(s.durationMs.median)}, p95 ${sec(s.durationMs.p95)}`
+    );
+  }
+  const cost = costPerPass(results);
+  out.push(
+    cost.kind === "unknown" ? "Cost per accepted change: unknown (a run's cost is unknown)" : cost.kind === "none" ? "Cost per accepted change: no passes" : `Cost per accepted change: ${cost.minimum ? "\u2265 " : ""}$${cost.usd.toFixed(4)} (all attempts ${cost.minimum ? "\u2265 " : ""}$${cost.totalUsd.toFixed(4)} / ${cost.passes} pass${cost.passes === 1 ? "" : "es"})`
+  );
+  const u = unexpectedChanges(results);
+  if (u.measured > 0) {
+    out.push(
+      `Unexpected changes: ${u.runs} of ${u.measured} runs, ${u.files} file${u.files === 1 ? "" : "s"}`
+    );
+    const shown = results.filter((r) => (r.unexpected?.length ?? 0) > 0).slice(0, 5);
+    for (const r of shown) {
+      out.push(
+        `  ${r.id}${r.attempt === void 0 ? "" : ` (run ${r.attempt})`}: ${(r.unexpected ?? []).join(", ")}`
+      );
+    }
+    if (u.runs > shown.length) out.push(`  \u2026 ${u.runs - shown.length} more in report.json`);
+  }
+  const mcp = mcpUse(results);
+  if (mcp !== void 0) out.push(`MCP tool calls: ${mcpUseText(mcp)}`);
+  const failures = failureLine(results);
+  if (failures !== void 0) out.push(failures);
+  return out;
+}
+function idWidth(results) {
+  return Math.max(18, ...results.map((r) => r.id.length));
+}
+function meanRows(results, ids) {
+  const width = idWidth(results);
+  return ids.map((id) => {
+    const all = results.filter((r) => r.id === id);
+    const runs = all.filter((r) => !isErrorRun(r));
+    const errors = all.length - runs.length;
+    const passesAll = all.filter((r) => r.passed).length;
+    const note = errors > 0 ? `  (${passesAll}/${all.length} of all attempts; ${errors} error run(s))` : "";
+    if (runs.length === 0) return `${"0/0".padEnd(4)}  ${id.padEnd(width)}${note}`;
+    const mean = (f) => runs.reduce((s, r) => s + f(r), 0) / runs.length;
+    const passes = runs.filter((r) => r.passed).length;
+    return [
+      `${passes}/${runs.length}`.padEnd(4),
+      id.padEnd(width),
+      `${mean((r) => r.steps).toFixed(1).padStart(5)} steps`,
+      runs.some((r) => (r.unreportedTokens ?? 0) > 0) ? "    tok ?" : `${(mean((r) => r.tokens) / 1e3).toFixed(1).padStart(7)}k tok`,
+      runs.every((r) => r.costUsd !== void 0) ? `${runs.some((r) => r.costIncomplete === true) ? "\u2265" : ""}$${mean((r) => r.costUsd ?? 0).toFixed(4).padStart(7)}` : "   cost ?",
+      ...stepsSd(runs) === void 0 ? [] : [`sd ${stepsSd(runs).toFixed(1)} steps`]
+    ].join("  ") + note;
+  });
+}
+var EVAL_BIOME_CONFIG = {
+  formatter: { indentStyle: "space", indentWidth: 2, lineWidth: 100 },
+  linter: { enabled: false },
+  assist: { enabled: false }
+};
+async function formatProject(root, biome) {
+  await writeFile4(join7(root, "biome.json"), `${JSON.stringify(EVAL_BIOME_CONFIG, null, 2)}
+`);
+  const result = await new HostExecutor().run(`${shellWord2(biome)} format --write .`, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: [...DEFAULT_ENV_ALLOWLIST],
+    timeoutMs: 6e4,
+    maxOutputBytes: 2e4
+  });
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `Biome could not format the eval project: ${result.stderr.text.trim().split("\n")[0] ?? ""}`
+    );
+  }
+}
+function shellWord2(text) {
+  return /^[A-Za-z0-9_./:@=+-]+$/.test(text) ? text : `'${text.replace(/'/g, "'\\''")}'`;
+}
+
+export {
+  DEFAULT_SEED,
+  shuffled,
+  byTask,
+  passInterval,
+  consistency,
+  spread,
+  stepsSd,
+  costPerPass,
+  unexpectedChanges,
+  loadReport,
+  compareReports,
+  formatComparison,
+  writeFiles,
+  MAVEN_POM,
+  MAVEN_CHECK,
+  PYPROJECT,
+  PYTEST_CHECK,
+  lines,
+  TOOLCHAINS,
+  checkToolchains,
+  prepareToolchain,
+  toolchainId,
+  DEFAULT_COMMITS,
+  buildRepoSuite,
+  saveRepoSuite,
+  loadRepoSuite,
+  repoEvalTasks,
+  EVAL_DENY_RULES,
+  REPO_CHECK_TIMEOUT_MS,
+  EVAL_TASK_TIMEOUT_MS,
+  PROVIDER_STALL_MS,
+  PROVIDER_STALLED,
+  MCP_PREPARE_TIMEOUT_MS,
+  ModelWatch,
+  repoTaskSandbox,
+  checkAsEval,
+  runEvalTask,
+  isAuthFailure,
+  isConnectionFailure,
+  saveUnique,
+  PROTECTED_FILES_CHANGED,
+  MCP_PREPARE_FAILED,
+  MCP_NOT_STARTED,
+  isMcpFailure,
+  isProviderStall,
+  isSetupFailure,
+  FAILURE_KINDS,
+  failureKind,
+  failureSummary,
+  failureLine,
+  withStalledRun,
+  runEvals,
+  runCheck,
+  isErrorRun,
+  summarize,
+  formatReport,
+  idWidth,
+  EVAL_BIOME_CONFIG,
+  formatProject
+};
+//# sourceMappingURL=chunk-A5DLAXB4.js.map