@garuda-agent/garuda 0.17.0 → 0.18.1

package/dist/evalCommand-BLJJ4G76.js

@@ -0,0 +1,696 @@
+#!/usr/bin/env node
+import {
+  EVAL_SUITES,
+  HELD_OUT_TASKS,
+  chooseTasks,
+  requiredToolchains
+} from "./chunk-GKU62MHQ.js";
+import {
+  formatReport,
+  isAuthFailure,
+  isMcpFailure,
+  isSetupFailure,
+  reportV2,
+  runEvals,
+  summarize
+} from "./chunk-DUYLVGPY.js";
+import {
+  DEFAULT_COMMITS,
+  DEFAULT_SEED,
+  TOOLCHAINS,
+  buildRepoSuite,
+  checkToolchains,
+  compareReports,
+  formatComparison,
+  loadRepoSuite,
+  loadReport,
+  prepareToolchain,
+  repoEvalTasks,
+  saveRepoSuite,
+  shuffled,
+  toolchainId
+} from "./chunk-CWVXOSAA.js";
+import {
+  BUILD_COMMIT
+} from "./chunk-H7LIBGZT.js";
+import {
+  garudaCommit,
+  staleBuildNote
+} from "./chunk-YAQ6QLKB.js";
+import "./chunk-NXJ4XLB5.js";
+import {
+  CODE_INDEX_MODES,
+  DEFAULT_CODE_INDEX_MODE,
+  detectProfiles
+} from "./chunk-5UDYQ5MK.js";
+import "./chunk-G2AX7MGS.js";
+import "./chunk-ULAJPMY7.js";
+import "./chunk-JMH5W5DG.js";
+import {
+  NO_MODEL,
+  chooseModel,
+  keepProviderKeys,
+  loadModelsConfig,
+  providerKeyValue,
+  resolveModel
+} from "./chunk-QLBCM2YQ.js";
+import "./chunk-NH72AELP.js";
+import "./chunk-4QRTTRVK.js";
+import "./chunk-A2QAIRQL.js";
+import {
+  newSessionId
+} from "./chunk-NLS7GMF7.js";
+import "./chunk-7KBOIPUT.js";
+import "./chunk-B25VL4RU.js";
+import "./chunk-MNE7HHX2.js";
+import "./chunk-IYGJWX65.js";
+import "./chunk-5DEWP3YX.js";
+import "./chunk-PZS3V6P3.js";
+import "./chunk-QANRZW7W.js";
+import "./chunk-ATW7BNIG.js";
+import "./chunk-WK5NE6RY.js";
+import {
+  detectTestCommand
+} from "./chunk-AOGFYUKG.js";
+import {
+  git
+} from "./chunk-QXTHEFKP.js";
+import {
+  EXECUTOR_NAMES,
+  HostExecutor,
+  createExecutor,
+  expandAllowlist
+} from "./chunk-LMKGSYJC.js";
+import "./chunk-BHZ6G65V.js";
+import "./chunk-P6XWWIEK.js";
+import "./chunk-UIRJOQQS.js";
+import "./chunk-4LFJFDCF.js";
+import "./chunk-3NJNVZTK.js";
+
+// src/cli/evalCommand.ts
+import { existsSync, mkdirSync, realpathSync, writeFileSync } from "fs";
+import { createRequire } from "module";
+import { homedir } from "os";
+import { delimiter, dirname, join } from "path";
+var BATCH_TASK_TIMEOUT_MS = 12 * 60 * 6e4;
+async function runEvalCommand(options) {
+  if (options.fromGit) return buildFromGit(options);
+  let repoTasks;
+  if (options.list || options.suite === "repo") {
+    const loaded = await repoSuite();
+    if (typeof loaded === "string") {
+      if (options.suite === "repo") {
+        process.stderr.write(`${loaded}
+`);
+        return 1;
+      }
+    } else repoTasks = repoEvalTasks(loaded);
+  }
+  if (options.list) {
+    const suites = [
+      ...Object.entries(EVAL_SUITES),
+      ...repoTasks === void 0 ? [] : [["repo", repoTasks]]
+    ];
+    for (const [suite2, tasks2] of suites) {
+      const needs2 = requiredToolchains(tasks2).map((id) => TOOLCHAINS[id].title);
+      process.stdout.write(`${suite2}:${needs2.length > 0 ? ` (needs ${needs2.join(", ")})` : ""}
+`);
+      for (const task of tasks2) process.stdout.write(`  ${task.id.padEnd(32)} ${task.title}
+`);
+    }
+    const needs = requiredToolchains(HELD_OUT_TASKS).map((id) => TOOLCHAINS[id].title);
+    process.stdout.write(
+      `
+held-out split (runs only with --split heldout or all; needs ${needs.join(", ")} for some):
+`
+    );
+    for (const task of HELD_OUT_TASKS) {
+      process.stdout.write(`  ${task.id.padEnd(32)} ${task.title}
+`);
+    }
+    return 0;
+  }
+  if (options.prepare !== void 0) return prepare(options.prepare);
+  const models = await loadModelsConfig();
+  if (models.problem !== void 0) {
+    process.stderr.write(`${models.problem}
+`);
+    return 1;
+  }
+  const modelId = chooseModel(models.config, options.model);
+  if (!modelId) {
+    process.stderr.write(`${NO_MODEL}
+`);
+    return 1;
+  }
+  let resolved;
+  try {
+    resolved = resolveModel(modelId, models.config);
+  } catch (error) {
+    process.stderr.write(`${error.message}
+`);
+    return 1;
+  }
+  for (const note of resolved.notes) process.stderr.write(`${note}
+`);
+  const index = options.index ?? DEFAULT_CODE_INDEX_MODE;
+  if (!CODE_INDEX_MODES.includes(index)) {
+    process.stderr.write(
+      `Unknown index mode ${options.index}. Use: ${CODE_INDEX_MODES.join(", ")}.
+`
+    );
+    return 1;
+  }
+  if (options.executor !== void 0 && !EXECUTOR_NAMES.includes(options.executor)) {
+    process.stderr.write(
+      `Unknown executor ${options.executor}. Use: ${EXECUTOR_NAMES.join(", ")}.
+`
+    );
+    return 1;
+  }
+  const subagentsMode = options.subagents ?? "off";
+  if (subagentsMode !== "on" && subagentsMode !== "off") {
+    process.stderr.write(`Unknown subagents mode ${options.subagents}. Use: on, off.
+`);
+    return 1;
+  }
+  const todoMode = options.todo ?? "off";
+  if (todoMode !== "on" && todoMode !== "off") {
+    process.stderr.write(`Unknown todo mode ${options.todo}. Use: on, off.
+`);
+    return 1;
+  }
+  const formatMode = options.format ?? "off";
+  if (formatMode !== "on" && formatMode !== "off") {
+    process.stderr.write(`Unknown format mode ${options.format}. Use: on, off.
+`);
+    return 1;
+  }
+  let biome;
+  if (options.format !== void 0) {
+    biome = garudaBiome();
+    if (biome === void 0) {
+      process.stderr.write(
+        "--format needs Biome in Garuda's node_modules (a source checkout after pnpm install).\n"
+      );
+      return 1;
+    }
+    if (options.suite === "java" || options.suite === "python" || options.suite === "all" || options.suite === "repo" || options.split !== void 0 && options.split !== "dev") {
+      process.stderr.write("--format works with the JS suites: basic and hard.\n");
+      return 1;
+    }
+  }
+  const thinkingMode = options.keepThinking ?? "on";
+  if (thinkingMode !== "on" && thinkingMode !== "off") {
+    process.stderr.write(`Unknown keep-thinking mode ${options.keepThinking}. Use: on, off.
+`);
+    return 1;
+  }
+  const lspMode = options.lsp ?? "off";
+  if (lspMode !== "on" && lspMode !== "off") {
+    process.stderr.write(`Unknown lsp mode ${options.lsp}. Use: on, off.
+`);
+    return 1;
+  }
+  let network;
+  if (options.network !== void 0) {
+    network = options.network.flatMap((e) => e.split(",")).filter((e) => e.trim() !== "");
+    const { problems } = expandAllowlist(network);
+    if (problems.length > 0) {
+      process.stderr.write(`--network: ${problems.join(" ")}
+`);
+      return 1;
+    }
+    if (network.length === 0) network = void 0;
+  }
+  const batchMode = options.batch ?? "off";
+  if (batchMode !== "on" && batchMode !== "off") {
+    process.stderr.write(`Unknown batch mode ${options.batch}. Use: on, off.
+`);
+    return 1;
+  }
+  const createBatch = resolved.createBatch;
+  if (batchMode === "on" && createBatch === void 0) {
+    process.stderr.write(`--batch on needs an Anthropic model; ${resolved.spec} is not one.
+`);
+    return 1;
+  }
+  let sub;
+  if (options.subagentModel) {
+    try {
+      sub = resolveModel(options.subagentModel, models.config);
+    } catch (error) {
+      process.stderr.write(`${error.message}
+`);
+      return 1;
+    }
+  }
+  const { assertModelAllowedByPolicy, loadTeamPolicy } = await import("./policy-ZJ7RMOFN.js");
+  let team;
+  try {
+    team = await loadTeamPolicy();
+    assertModelAllowedByPolicy(team?.policy, resolved.spec);
+    if (sub !== void 0) assertModelAllowedByPolicy(team?.policy, sub.spec);
+  } catch (error) {
+    process.stderr.write(`Team policy: ${error.message}
+`);
+    return 1;
+  }
+  const credentialsWarning = await keepProviderKeys([
+    resolved.def.apiKeyEnv,
+    sub?.def.apiKeyEnv,
+    "ANTHROPIC_API_KEY"
+  ]);
+  if (credentialsWarning !== void 0) process.stderr.write(`${credentialsWarning}
+`);
+  const agentName = options.agent ?? "garuda";
+  let baseline;
+  if (agentName === "claude-code") {
+    const made = await claudeCodeAgent(options, resolved);
+    if (typeof made === "string") {
+      process.stderr.write(`--agent claude-code: ${made}
+`);
+      return 1;
+    }
+    baseline = made;
+    network = ["api.anthropic.com"];
+  } else if (agentName !== "garuda") {
+    process.stderr.write(`Unknown agent ${options.agent}. Use: garuda, claude-code.
+`);
+    return 1;
+  }
+  const split = options.split ?? "dev";
+  const suite = options.suite ?? (split === "dev" ? "basic" : split === "all" ? "all" : "heldout");
+  const choice = chooseTasks({
+    suite: options.suite,
+    split,
+    task: options.task,
+    repoTasks
+  });
+  if ("problem" in choice) {
+    process.stderr.write(`${choice.problem}
+`);
+    return 1;
+  }
+  const chosen = choice.tasks;
+  const repeat = Math.max(1, options.repeat ?? 1);
+  const seed = options.seed ?? DEFAULT_SEED;
+  if (!Number.isInteger(seed) || seed < 0) {
+    process.stderr.write("--seed must be a whole number, 0 or more.\n");
+    return 1;
+  }
+  const listed = chosen.flatMap((t) => Array.from({ length: repeat }, () => t));
+  const tasks = options.shuffle ? shuffled(listed, seed) : listed;
+  if (chosen.length === 0) {
+    process.stderr.write("No task matches. Use --list to see the task ids.\n");
+    return 1;
+  }
+  const missing = (await checkToolchains(requiredToolchains(chosen))).filter((s) => !s.ok);
+  if (missing.length > 0) {
+    for (const status of missing) {
+      const detail = status.output.split("\n").filter((line) => line.trim() !== "").slice(0, 2).join("\n    ");
+      process.stderr.write(
+        `${TOOLCHAINS[status.id].hint}
+${detail === "" ? "" : `    ${detail}
+`}`
+      );
+    }
+    return 1;
+  }
+  const mcp = await chooseMcp(options, homedir());
+  if (typeof mcp === "string") {
+    process.stderr.write(`${mcp}
+`);
+    return 1;
+  }
+  let addendum;
+  if (options.addendum !== void 0) {
+    const { readAddendum } = await import("./tune-VX2WQTHS.js");
+    const read = await readAddendum(options.addendum);
+    if ("problem" in read) {
+      process.stderr.write(`${read.problem}
+`);
+      return 1;
+    }
+    addendum = read.text;
+  }
+  const { executor, notice } = createExecutor(options.executor ?? "auto");
+  if (notice !== void 0) process.stderr.write(`${notice}
+`);
+  if (baseline !== void 0 && executor.isolation === "none") {
+    process.stderr.write(
+      "--agent claude-code needs the OS sandbox: Claude Code runs with every tool allowed, so Garuda runs it only inside the sandbox.\n"
+    );
+    return 1;
+  }
+  if (lspMode === "on") {
+    if (executor.isolation === "none") {
+      process.stderr.write("--lsp on needs the OS sandbox: language servers run only there.\n");
+      return 1;
+    }
+    const { discoverServer, LSP_LANGUAGES } = await import("./servers-6YDGWOZ6.js");
+    const found = LSP_LANGUAGES.map((l) => ({ l, s: discoverServer(l, { root: process.cwd() }) }));
+    if (found.every((f) => f.s === void 0)) {
+      process.stderr.write(
+        "--lsp on: no language server found. Run: garuda lsp install typescript\n"
+      );
+      return 1;
+    }
+    for (const { l, s } of found) {
+      process.stderr.write(
+        `LSP ${l}: ${s === void 0 ? "none" : `${s.spec.name} (${s.path})`}
+`
+      );
+    }
+  }
+  const where = executor.isolation === "none" ? "commands run on this machine with no sandbox" : `commands and the checks run in the ${executor.name} sandbox`;
+  const parallel = Math.max(
+    1,
+    options.parallel ?? (batchMode === "on" ? Math.min(tasks.length, 20) : 1)
+  );
+  const outDir = options.outDir ?? join(process.cwd(), ".garuda", "evals", newSessionId());
+  mkdirSync(outDir, { recursive: true });
+  const stale = staleBuildNote(BUILD_COMMIT, await garudaCommit());
+  if (stale !== void 0) process.stderr.write(`Warning: ${stale}
+
+`);
+  process.stderr.write(
+    `${baseline === void 0 ? "" : `Baseline: Claude Code ${baseline.version ?? ""} (claude -p, every tool allowed, in the sandbox, network only to api.anthropic.com).
+`}Running ${chosen.length} task(s)${repeat > 1 ? ` \xD7 ${repeat}` : ""}${options.shuffle ? ` in a shuffled order (seed ${seed})` : ""} (suite ${options.task === void 0 ? suite : "custom"}, code index ${index}, subagents ${subagentsMode}${sub === void 0 ? "" : ` on ${sub.spec}`}, todo ${todoMode}, lsp ${lspMode}, keep-thinking ${thinkingMode}, format ${formatMode}, network ${network === void 0 ? "off" : network.join(",")}, mcp ${mcp === void 0 ? "off" : `${mcp.server}${mcp.prepare === void 0 ? "" : `, prepared with \`${mcp.prepare}\``}`}, batch ${batchMode}${addendum === void 0 ? "" : `, addendum ${options.addendum} (${addendum.length} characters)`}${parallel > 1 ? `, ${parallel} at a time` : ""}) with ${modelId}. Garuda approves every call except its deny rules;
+${where}, in scratch folders.
+
+`
+  );
+  if (batchMode === "on") {
+    process.stderr.write(
+      "Batch API: each model call waits for its batch (minutes, sometimes more) at half the token price.\n\n"
+    );
+  }
+  const started = Date.now();
+  const results = await runEvals(
+    tasks,
+    {
+      modelId: resolved.spec,
+      ...team === void 0 ? {} : { policy: team.policy },
+      model: () => batchMode === "on" && createBatch !== void 0 ? createBatch() : resolved.create(),
+      // Batch responses carry their own price factor (half): the price stays the model's.
+      modelInfo: resolved.info,
+      parallel,
+      ...batchMode === "on" ? { taskTimeoutMs: BATCH_TASK_TIMEOUT_MS } : {},
+      ...resolved.maxTokens === void 0 ? {} : { maxTokens: resolved.maxTokens },
+      outDir,
+      ...options.maxSteps === void 0 ? {} : { maxSteps: options.maxSteps },
+      codeIndex: index,
+      subagents: subagentsMode === "on",
+      todo: todoMode === "on",
+      lsp: lspMode === "on",
+      keepThinking: thinkingMode === "on",
+      ...network === void 0 ? {} : { network },
+      ...baseline === void 0 ? {} : { agent: baseline },
+      ...mcp === void 0 ? {} : { mcp },
+      ...addendum === void 0 ? {} : { addendum },
+      ...biome === void 0 ? {} : { format: { on: formatMode === "on", biome } },
+      ...sub === void 0 ? {} : { subagentModel: { spec: sub.spec, model: () => sub.create(), info: sub.info } },
+      executor: executor.name === "host" ? "host" : "os",
+      ...options.keep ? { keep: true } : {},
+      onEvent: (taskId, event) => {
+        if (event.type === "tool_call") process.stderr.write(`  [${taskId}] ${event.call.name}
+`);
+        if (event.type === "notice") process.stderr.write(`  [${taskId}] ${event.text}
+`);
+        if (event.type === "model_retry")
+          process.stderr.write(
+            `  [${taskId}] retry ${event.attempt}/${event.maxRetries}: ${event.reason}
+`
+          );
+      }
+    },
+    (r) => process.stderr.write(
+      `${r.passed ? "\u2713" : "\u2717"} ${r.id}: ${r.passed ? "pass" : "fail"} (${r.steps} steps)${r.reason ? `
+    ${r.reason.split("\n").slice(0, 3).join("\n    ")}` : ""}
+`
+    ),
+    // A refused key or an unreachable API fails every run: stop at the first one (0.17).
+    isSetupFailure
+  );
+  const wallMs = Date.now() - started;
+  if (results.length < tasks.length) {
+    const last = results[results.length - 1];
+    process.stderr.write(
+      last !== void 0 && isAuthFailure(last) ? `
+Stopped after ${results.length} of ${tasks.length} run(s): the provider refused the API key or the access. Check ANTHROPIC_API_KEY (it wins over the stored key) or run \`garuda setup\`.
+` : last !== void 0 && isMcpFailure(last) ? `
+Stopped after ${results.length} of ${tasks.length} run(s): the --mcp server or its prepare command failed (see the reason above), so every other run would fail the same way.
+` : `
+Stopped after ${results.length} of ${tasks.length} run(s): the API could not be reached (see the reason above), so every other run would fail the same way.
+`
+    );
+  }
+  const report = `${formatReport(results, { seed })}
+Wall time: ${Math.round(wallMs / 1e3)} s${parallel > 1 ? ` (${parallel} tasks at a time)` : ""}.`;
+  writeFileSync(
+    join(outDir, "report.json"),
+    `${JSON.stringify({ ...await reportV2({ model: resolved.spec, provider: resolved.provider, executor: executor.name, isolation: executor.isolation, seed, shuffle: options.shuffle === true, split, ...mcp === void 0 ? {} : { mcp }, ...addendum === void 0 ? {} : { addendum: { file: options.addendum ?? "", text: addendum } }, agent: baseline === void 0 ? { name: "garuda" } : { name: "claude-code", ...baseline.version === void 0 ? {} : { version: baseline.version } } }, chosen, results), model: modelId, suite, repeat, codeIndex: index, subagents: subagentsMode, subagentModel: sub?.spec, todo: todoMode, lsp: lspMode, keepThinking: thinkingMode, format: formatMode, network: network ?? "off", batch: batchMode, parallel, wallMs, executor: executor.name, summary: summarize(results), results }, null, 2)}
+`
+  );
+  process.stdout.write(`
+${report}
+
+Session files and report.json: ${outDir}
+`);
+  return results.every((r) => r.passed) ? 0 : 2;
+}
+async function repoSuite() {
+  const root = await gitRoot(process.cwd());
+  if (root === void 0) return "-s repo needs a git repository: run it in your project.";
+  let suite;
+  try {
+    suite = await loadRepoSuite(root);
+  } catch (error) {
+    return error.message;
+  }
+  if (suite === void 0) {
+    return `No repo suite in ${root}. Build it first: garuda eval --from-git`;
+  }
+  if (suite.tasks.length === 0) {
+    return `The repo suite has no tasks. Build it again with more commits: garuda eval --from-git --commits 500`;
+  }
+  return { ...suite, root };
+}
+async function gitRoot(cwd) {
+  const r = await git(new HostExecutor(), cwd, ["rev-parse", "--show-toplevel"], { check: false });
+  return r.exitCode === 0 ? realpathSync(r.stdout.trim()) : void 0;
+}
+async function buildFromGit(options) {
+  const root = await gitRoot(process.cwd());
+  if (root === void 0) {
+    process.stderr.write("--from-git needs a git repository: run it in your project.\n");
+    return 1;
+  }
+  const testCommand = options.testCommand ?? detectTestCommand(root, detectProfiles(root));
+  if (testCommand === void 0) {
+    process.stderr.write(
+      'No test command found. Give one: garuda eval --from-git --test-command "npm test"\n'
+    );
+    return 1;
+  }
+  for (const [name, value] of [
+    ["--commits", options.commits],
+    ["--max-tasks", options.maxTasks]
+  ]) {
+    if (value !== void 0 && !(Number.isInteger(value) && value > 0)) {
+      process.stderr.write(`${name} must be a whole number above 0.
+`);
+      return 1;
+    }
+  }
+  process.stderr.write(
+    `Building the repo suite from ${options.commits ?? DEFAULT_COMMITS} recent commits${options.since === void 0 ? "" : ` since ${options.since}`} in ${root}.
+Each candidate runs its own test files with \`${testCommand}\` twice on this machine (no sandbox), in a git worktree; your checkout does not change.
+
+`
+  );
+  const { suite, checked, skipped, failure } = await buildRepoSuite(new HostExecutor(), root, {
+    testCommand,
+    ...options.commits === void 0 ? {} : { commits: options.commits },
+    ...options.since === void 0 ? {} : { since: options.since },
+    ...options.maxTasks === void 0 ? {} : { maxTasks: options.maxTasks },
+    onProgress: (line) => process.stderr.write(`  ${line}
+`)
+  });
+  const file = await saveRepoSuite(suite);
+  const reasons = Object.entries(skipped).sort((a, b) => b[1] - a[1]).map(([why, n]) => `  ${String(n).padStart(4)}  ${why}`);
+  process.stdout.write(
+    `${[
+      `
+${suite.tasks.length} task(s) kept; ${checked} candidate(s) checked with the tests.`,
+      ...reasons.length === 0 ? [] : ["Skipped:", ...reasons],
+      ...failure === void 0 ? [] : [
+        `The first failure at a commit (${failure.sha.slice(0, 7)}: \`${failure.command}\`):`,
+        ...failure.tail.split("\n").map((line) => `  ${line}`),
+        'If every commit fails like this, the tests do not run in a worktree on this machine; give a working command with --test-command ("{files}" takes the test files).'
+      ],
+      `Suite: ${file}`,
+      ...suite.tasks.length === 0 ? ["No task: try more commits (--commits 500) or an older --since."] : ["Run it: garuda eval -s repo -m <model> (add --list to see the tasks)"]
+    ].join("\n")}
+`
+  );
+  return suite.tasks.length === 0 ? 2 : 0;
+}
+async function prepare(name) {
+  const id = toolchainId(name);
+  if (id === void 0) {
+    process.stderr.write(`Unknown toolchain ${name}. Use: java, python.
+`);
+    return 1;
+  }
+  const toolchain = TOOLCHAINS[id];
+  if (toolchain.prepare !== void 0) {
+    process.stderr.write(
+      `Preparing ${toolchain.title}: running \`${toolchain.prepare}\` in a scratch project (downloads with network)\u2026
+`
+    );
+    const r = await prepareToolchain(id);
+    if (!r.ok) {
+      process.stderr.write(`It failed:
+${r.output}
+`);
+      return 1;
+    }
+  }
+  const [status] = await checkToolchains([id]);
+  if (status?.ok) {
+    process.stderr.write(`${toolchain.title}: ready.
+`);
+    return 0;
+  }
+  process.stderr.write(`${toolchain.hint}
+${status?.output ?? ""}
+`);
+  return 1;
+}
+function garudaBiome() {
+  try {
+    const pkg = createRequire(import.meta.url).resolve("@biomejs/biome/package.json");
+    const bin = join(dirname(pkg), "bin", "biome");
+    return existsSync(bin) ? bin : void 0;
+  } catch {
+    return void 0;
+  }
+}
+async function runEvalCompare(pathA, pathB, options = {}) {
+  const seed = options.seed ?? DEFAULT_SEED;
+  if (!Number.isInteger(seed) || seed < 0) {
+    process.stderr.write("--seed must be a whole number, 0 or more.\n");
+    return 1;
+  }
+  let a;
+  let b;
+  try {
+    [a, b] = await Promise.all([loadReport(pathA), loadReport(pathB)]);
+  } catch (error) {
+    process.stderr.write(`garuda eval compare: ${error.message}
+`);
+    return 1;
+  }
+  const result = compareReports(a, b, { seed });
+  if ("problems" in result) {
+    process.stderr.write(
+      `garuda eval compare: these reports cannot be compared.
+${result.problems.map((p) => `  ${p}`).join("\n")}
+`
+    );
+    return 1;
+  }
+  process.stdout.write(`${formatComparison(a, b, result.comparison)}
+`);
+  return 0;
+}
+async function claudeCodeAgent(options, resolved) {
+  if (resolved.provider !== "anthropic") {
+    return `the baseline needs a Claude model; ${resolved.spec} is not one.`;
+  }
+  const garudaOnly = [
+    ["--index", options.index],
+    ["--subagents", options.subagents],
+    ["--subagent-model", options.subagentModel],
+    ["--todo", options.todo],
+    ["--lsp", options.lsp],
+    ["--keep-thinking", options.keepThinking],
+    ["--format", options.format],
+    ["--batch", options.batch],
+    ["--network", options.network],
+    ["--mcp", options.mcp],
+    ["--mcp-prepare", options.mcpPrepare],
+    ["--addendum", options.addendum]
+  ];
+  const set = garudaOnly.filter(([, value]) => value !== void 0).map(([name]) => name);
+  if (set.length > 0) {
+    return `${set.join(", ")} change${set.length === 1 ? "s" : ""} Garuda only; leave ${set.length === 1 ? "it" : "them"} out of the baseline run.`;
+  }
+  const key = providerKeyValue("ANTHROPIC_API_KEY");
+  if (key === void 0 || key === "") {
+    return "set ANTHROPIC_API_KEY (or store a key with `garuda setup`). A Claude subscription login is not used: the sandbox keeps the Keychain closed.";
+  }
+  const command = onPath("claude");
+  if (command === void 0) {
+    return "the `claude` program is not on PATH. Install Claude Code: npm install -g @anthropic-ai/claude-code";
+  }
+  const probe = await new HostExecutor().run(`'${command.replace(/'/g, "'\\''")}' --version`, {
+    root: process.cwd(),
+    sandbox: false,
+    writePaths: [],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: ["PATH", "HOME"],
+    timeoutMs: 2e4,
+    maxOutputBytes: 2e3
+  });
+  const version = probe.exitCode === 0 ? probe.stdout.text.trim().split("\n")[0] : void 0;
+  return {
+    name: "claude-code",
+    command,
+    model: resolved.model,
+    ...version === void 0 || version === "" ? {} : { version },
+    env: { ANTHROPIC_API_KEY: key }
+  };
+}
+function onPath(program) {
+  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
+    const path = join(dir, program);
+    if (dir !== "" && existsSync(path)) return path;
+  }
+  return void 0;
+}
+async function chooseMcp(options, home) {
+  if (options.mcp === void 0) {
+    return options.mcpPrepare === void 0 ? void 0 : "--mcp-prepare needs --mcp <server>.";
+  }
+  const { loadMcpConfig } = await import("./config-WV2H64TF.js");
+  const { servers, problems } = await loadMcpConfig({ home, root: home });
+  const own = servers.filter((s) => s.source === "user");
+  const server = own.find((s) => s.name === options.mcp);
+  if (server === void 0) {
+    const names = own.map((s) => s.name);
+    return `--mcp ${options.mcp}: no server of that name in ~/.garuda/mcp.json${names.length === 0 ? "" : ` (yours: ${names.join(", ")})`}.${problems.length === 0 ? "" : `
+${problems.join("\n")}`}`;
+  }
+  if (!server.def.enabled) {
+    return `--mcp ${options.mcp}: the server is disabled ("enabled": false) in ~/.garuda/mcp.json.`;
+  }
+  return {
+    server: server.name,
+    home,
+    ...options.mcpPrepare === void 0 ? {} : { prepare: options.mcpPrepare }
+  };
+}
+export {
+  BATCH_TASK_TIMEOUT_MS,
+  chooseMcp,
+  claudeCodeAgent,
+  repoSuite,
+  runEvalCommand,
+  runEvalCompare
+};
+//# sourceMappingURL=evalCommand-BLJJ4G76.js.map