@garuda-agent/garuda 0.18.0 → 0.18.1

package/dist/chunk-DUYLVGPY.js

@@ -0,0 +1,821 @@
+#!/usr/bin/env node
+import {
+  DEFAULT_SEED,
+  TOOLCHAIN_ENV,
+  applyTests,
+  byTask,
+  consistency,
+  costPerPass,
+  isProtectedPath,
+  isolatedTaskCopy,
+  mcpUse,
+  mcpUseText,
+  passInterval,
+  repoTaskReadDenials,
+  roughNote,
+  spread,
+  stepsSd,
+  unexpectedChanges,
+  writeFiles
+} from "./chunk-CWVXOSAA.js";
+import {
+  BUILD_COMMIT
+} from "./chunk-H7LIBGZT.js";
+import {
+  garudaCommit
+} from "./chunk-YAQ6QLKB.js";
+import {
+  AutoApprover
+} from "./chunk-NXJ4XLB5.js";
+import {
+  DEFAULT_MAX_STEPS,
+  Runtime
+} from "./chunk-5UDYQ5MK.js";
+import {
+  FileSessionStore,
+  isCostIncomplete
+} from "./chunk-NLS7GMF7.js";
+import {
+  totalTokens
+} from "./chunk-7KBOIPUT.js";
+import {
+  VERSION
+} from "./chunk-IYGJWX65.js";
+import {
+  git
+} from "./chunk-QXTHEFKP.js";
+import {
+  DEFAULT_ENV_ALLOWLIST,
+  HostExecutor,
+  parseSettings
+} from "./chunk-LMKGSYJC.js";
+import {
+  pathMatches
+} from "./chunk-3NJNVZTK.js";
+
+// src/evals/runner.ts
+import { constants as fsConstants, mkdirSync, realpathSync } from "fs";
+import { copyFile, mkdtemp as mkdtemp2, readFile as readFile2, rm as rm2, writeFile as writeFile2 } from "fs/promises";
+import { homedir, tmpdir as tmpdir2 } from "os";
+import { join as join3 } from "path";
+
+// src/evals/agents/claudeCode.ts
+import { mkdtemp, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+var MAX_OUTPUT_BYTES = 5e6;
+function claudeCommand(agent, prompt, maxTurns) {
+  return [
+    quote(agent.command),
+    "-p",
+    quote(prompt),
+    "--output-format",
+    "json",
+    "--model",
+    quote(agent.model),
+    "--max-turns",
+    String(maxTurns),
+    "--permission-mode",
+    "bypassPermissions"
+  ].join(" ");
+}
+function parseClaudeResult(stdout) {
+  const line = stdout.trim().split("\n").reverse().find((l) => l.trim().startsWith("{"));
+  let data;
+  try {
+    data = JSON.parse(line ?? "");
+  } catch {
+    return {
+      stopReason: "error",
+      steps: 0,
+      tokens: 0,
+      cacheReadTokens: 0,
+      costUsd: void 0,
+      reason: `Claude Code gave no JSON result: ${stdout.trim().slice(0, 300) || "no output"}`
+    };
+  }
+  const usage = data.usage ?? {};
+  const n = (v) => typeof v === "number" && Number.isFinite(v) ? v : 0;
+  const cacheRead = n(usage.cache_read_input_tokens);
+  const tokens = n(usage.input_tokens) + n(usage.output_tokens) + cacheRead + n(usage.cache_creation_input_tokens);
+  const costUsd = typeof data.total_cost_usd === "number" ? data.total_cost_usd : void 0;
+  const steps = n(data.num_turns);
+  const subtype = typeof data.subtype === "string" ? data.subtype : "";
+  const base = { steps, tokens, cacheReadTokens: cacheRead, costUsd };
+  if (subtype === "error_max_turns") return { ...base, stopReason: "max_steps" };
+  if (subtype === "success" && data.is_error !== true) return { ...base, stopReason: "done" };
+  const text = typeof data.result === "string" ? data.result : subtype || "unknown error";
+  return { ...base, stopReason: "error", reason: `Claude Code failed: ${text.slice(0, 300)}` };
+}
+async function runClaudeCode(executor, policy, agent, prompt, maxTurns, signal) {
+  const config = await mkdtemp(join(tmpdir(), "garuda-claude-config-"));
+  try {
+    await writeFile(
+      join(config, ".claude.json"),
+      `${JSON.stringify({ hasCompletedOnboarding: true })}
+`
+    );
+    const result = await executor.run(
+      claudeCommand(agent, prompt, maxTurns),
+      { ...policy, maxOutputBytes: MAX_OUTPUT_BYTES },
+      {
+        signal,
+        env: {
+          ...agent.env,
+          CLAUDE_CONFIG_DIR: config,
+          CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
+          DISABLE_AUTOUPDATER: "1"
+        }
+      }
+    );
+    const outcome = result.timedOut || result.aborted ? {
+      stopReason: "timeout",
+      steps: 0,
+      tokens: 0,
+      cacheReadTokens: 0,
+      costUsd: void 0,
+      reason: "Claude Code took longer than the task's time limit."
+    } : parseClaudeResult(result.stdout.text);
+    if (outcome.stopReason === "error" && result.stderr.text.trim() !== "" && outcome.reason !== void 0) {
+      outcome.reason += `
+${result.stderr.text.trim().split("\n").slice(-3).join("\n")}`;
+    }
+    return { result, outcome };
+  } finally {
+    await rm(config, { recursive: true, force: true });
+  }
+}
+function quote(text) {
+  return `'${text.replace(/'/g, "'\\''")}'`;
+}
+
+// src/evals/changes.ts
+import { createHash } from "crypto";
+import { lstat, readdir, readFile, readlink } from "fs/promises";
+import { join as join2 } from "path";
+var IGNORED_DIRS = /* @__PURE__ */ new Set([
+  ".git",
+  ".garuda",
+  // Claude Code's own project folder (the T11b baseline), as .garuda is Garuda's.
+  ".claude",
+  "node_modules",
+  "target",
+  "build",
+  "dist",
+  "__pycache__",
+  ".pytest_cache"
+]);
+var IGNORED_FILE = /\.(class|pyc)$/;
+var MAX_TRACKED_FILES = 5e4;
+async function treeState(root) {
+  const out = /* @__PURE__ */ new Map();
+  const walk = async (dir, prefix) => {
+    let names;
+    try {
+      names = await readdir(dir);
+    } catch {
+      return;
+    }
+    for (const name of names.sort()) {
+      if (out.size >= MAX_TRACKED_FILES) return;
+      const path = prefix === "" ? name : `${prefix}/${name}`;
+      const full = join2(dir, name);
+      const info = await lstat(full).catch(() => void 0);
+      if (info === void 0) continue;
+      if (info.isDirectory()) {
+        if (!IGNORED_DIRS.has(name)) await walk(full, path);
+      } else if (info.isSymbolicLink()) {
+        out.set(path, `link:${await readlink(full).catch(() => "")}`);
+      } else if (info.isFile() && !IGNORED_FILE.test(name)) {
+        const hash = createHash("sha256");
+        hash.update(await readFile(full).catch(() => Buffer.alloc(0)));
+        out.set(path, hash.digest("hex"));
+      }
+    }
+  };
+  await walk(root, "");
+  return out;
+}
+function changedPaths(before, after) {
+  const out = /* @__PURE__ */ new Set();
+  for (const [path, hash] of after) if (before.get(path) !== hash) out.add(path);
+  for (const path of before.keys()) if (!after.has(path)) out.add(path);
+  return [...out].sort();
+}
+function unexpectedPaths(changed, expected, allowChanges = []) {
+  const known = new Set(expected);
+  return changed.filter(
+    (path) => !known.has(path) && !allowChanges.some((glob) => pathMatches(glob, path, false))
+  );
+}
+
+// src/evals/runner.ts
+var EVAL_DENY_RULES = [
+  "bash(rm -rf*)",
+  "bash(sudo*)",
+  "bash(git push*)",
+  "bash(curl*)",
+  "bash(wget*)"
+];
+var REPO_CHECK_TIMEOUT_MS = 5 * 6e4;
+var EVAL_TASK_TIMEOUT_MS = 10 * 6e4;
+var MCP_PREPARE_TIMEOUT_MS = 12e4;
+async function runEvalTask(task, options) {
+  const started = Date.now();
+  const host = new HostExecutor();
+  let root;
+  if (task.repo === void 0) {
+    root = realpathSync(await mkdtemp2(join3(tmpdir2(), `garuda-eval-${task.id}-`)));
+    await writeFiles(root, task.files);
+  } else {
+    root = await isolatedTaskCopy(host, task.repo.root, task.repo.base);
+    await applyTests(root, task.repo.tests);
+  }
+  if (options.format !== void 0) await formatProject(root, options.format.biome);
+  const protect = task.protect ?? Object.keys(task.files).filter(isProtectedPath);
+  const before = await snapshot(root, protect);
+  const settings = parseSettings({
+    // With an allowlist, a host outside it is denied: nobody answers questions in an eval.
+    permissions: {
+      deny: options.network === void 0 ? EVAL_DENY_RULES : [...EVAL_DENY_RULES, "network"]
+    },
+    ...options.network === void 0 ? {} : { network: { allow: options.network } },
+    // Evals must not depend on the internet.
+    web: { enabled: false },
+    // 0.18.1: a repo task's commands cannot read the project's own folder or the rest of the
+    // home folder: the checkout, and copies of it, hold the commit that the task asks for.
+    ...task.repo === void 0 ? {} : { sandbox: { denyRead: repoTaskReadDenials(task.repo.root, options.home ?? homedir()) } },
+    ...options.executor === void 0 ? {} : { executor: options.executor },
+    ...options.codeIndex === void 0 ? {} : { codeIndex: options.codeIndex },
+    ...options.maxSteps === void 0 ? {} : { limits: { maxSteps: options.maxSteps } },
+    ...options.subagents === void 0 ? {} : { subagents: { enabled: options.subagents } },
+    ...options.todo === void 0 ? {} : { todo: { enabled: options.todo } },
+    ...options.format === void 0 ? {} : {
+      formatters: {
+        enabled: options.format.on,
+        commands: {
+          biome: {
+            extensions: ["js", "jsx", "mjs", "cjs", "ts", "tsx", "json"],
+            command: [options.format.biome, "format", "--write", "$FILE"]
+          }
+        }
+      }
+    },
+    ...options.keepThinking === void 0 ? {} : { thinking: { keepBlocks: options.keepThinking } }
+  });
+  const store = new FileSessionStore(root);
+  let mcpCalls = 0;
+  const tools = [];
+  const createRuntime = (prepare2) => Runtime.create({
+    root,
+    modelId: options.modelId,
+    model: async () => options.model(),
+    ...options.modelInfo === void 0 ? {} : { modelInfo: options.modelInfo },
+    ...options.maxTokens === void 0 ? {} : { maxTokens: options.maxTokens },
+    ...options.subagentModel === void 0 ? {} : { subagentModel: options.subagentModel },
+    approver: new AutoApprover("once"),
+    store,
+    settings,
+    // The user's own MCP servers must not change eval results; `--mcp` allows one by name.
+    mcp: options.mcp === void 0 || prepare2 ? false : { home: options.mcp.home, only: options.mcp.server },
+    hooks: false,
+    ...options.addendum === void 0 ? {} : { addendum: options.addendum },
+    ...options.policy === void 0 ? {} : { policy: options.policy },
+    commands: false,
+    // The list comes from the eval command line: the user typed it.
+    ...options.network === void 0 ? {} : { network: { approved: true } },
+    ...options.lsp === void 0 ? {} : { lsp: { enabled: options.lsp } },
+    onEvent: (event) => {
+      if (event.type === "tool_call") {
+        tools.push(event.call.name);
+        if (event.call.name.startsWith("mcp__")) mcpCalls++;
+      }
+      options.onEvent?.(task.id, event);
+    }
+  });
+  let prepareFailure;
+  const prepare = options.mcp?.prepare;
+  if (prepare !== void 0) {
+    const prep = await createRuntime(true);
+    try {
+      const prepared = await runCheck(
+        root,
+        prepare,
+        MCP_PREPARE_TIMEOUT_MS,
+        prep.executor.isolation === "none" ? void 0 : { executor: prep.executor, policy: prep.execPolicy(MCP_PREPARE_TIMEOUT_MS) }
+      );
+      if (!prepared.ok) prepareFailure = `${MCP_PREPARE_FAILED}
+${prepared.output}`;
+    } finally {
+      await prep.close();
+      prep.executor.shutdown();
+    }
+  }
+  const runtime = await createRuntime(false);
+  const result = {
+    id: task.id,
+    title: task.title,
+    passed: false,
+    stopReason: "error",
+    steps: 0,
+    tokens: 0,
+    costUsd: void 0,
+    durationMs: 0
+  };
+  if (prepareFailure !== void 0) result.reason = prepareFailure;
+  const start = await treeState(root);
+  const limitMs = options.taskTimeoutMs ?? EVAL_TASK_TIMEOUT_MS;
+  const timeout = AbortSignal.timeout(limitMs);
+  try {
+    if (result.reason !== void 0) {
+    } else if (options.agent !== void 0) {
+      await runtime.prepareCommands(timeout);
+      const { result: exec, outcome } = await runClaudeCode(
+        runtime.executor,
+        runtime.execPolicy(limitMs),
+        options.agent,
+        task.prompt,
+        options.maxSteps ?? DEFAULT_MAX_STEPS,
+        timeout
+      );
+      result.stopReason = outcome.stopReason;
+      result.steps = outcome.steps;
+      result.tokens = outcome.tokens;
+      result.cacheReadTokens = outcome.cacheReadTokens;
+      result.costUsd = outcome.costUsd;
+      if (outcome.reason !== void 0) result.reason = outcome.reason;
+      if (options.outDir !== void 0) {
+        mkdirSync(options.outDir, { recursive: true });
+        result.sessionFile = await saveUnique(
+          options.outDir,
+          task.id,
+          ".claude.json",
+          (path) => writeFile2(path, exec.stdout.text, { flag: "wx" })
+        );
+      }
+    } else {
+      const run = await runtime.runTurn(task.prompt, timeout);
+      result.stopReason = run.stopReason;
+      result.steps = run.steps;
+    }
+  } catch (error) {
+    runtime.recordStop("error");
+    if (timeout.aborted) {
+      result.stopReason = "timeout";
+      result.reason = `The task took longer than ${limitMs / 6e4} minutes.`;
+    } else {
+      result.reason = `The run failed: ${error.message}`;
+    }
+  } finally {
+    if (tools.length > 0) result.tools = tools;
+    if (options.mcp !== void 0) {
+      result.mcpCalls = mcpCalls;
+      const status = runtime.mcpStatus().find((m) => m.name === options.mcp?.server);
+      if (result.reason === void 0 && status !== void 0 && status.state === "failed") {
+        result.stopReason = "error";
+        result.reason = `${MCP_NOT_STARTED} "${options.mcp.server}": ${status.message ?? "no reason given"}`;
+      }
+    }
+    await runtime.close();
+    runtime.executor.shutdown();
+  }
+  const session = runtime.session;
+  if (session !== void 0 && options.agent === void 0) {
+    result.tokens = totalTokens(session.usage);
+    if (session.unreportedTokens > 0) result.unreportedTokens = session.unreportedTokens;
+    result.costUsd = session.costUsd;
+    if (isCostIncomplete(session)) result.costIncomplete = true;
+    result.cacheReadTokens = session.usage.cacheReadTokens;
+    if (options.outDir !== void 0) {
+      mkdirSync(options.outDir, { recursive: true });
+      const from = store.path(session.id);
+      result.sessionFile = await saveUnique(
+        options.outDir,
+        task.id,
+        ".jsonl",
+        (path) => copyFile(from, path, fsConstants.COPYFILE_EXCL)
+      );
+    }
+  }
+  result.changed = changedPaths(start, await treeState(root));
+  const expected = await expectedFiles(task, host);
+  if (expected !== void 0) {
+    result.unexpected = unexpectedPaths(result.changed, expected, task.allowChanges);
+  }
+  if (result.reason === void 0) {
+    const changed = await changedFiles(root, before);
+    const checkMs = task.repo === void 0 ? 12e4 : REPO_CHECK_TIMEOUT_MS;
+    const check = await runCheck(
+      root,
+      task.check,
+      checkMs,
+      runtime.executor.isolation === "none" ? void 0 : { executor: runtime.executor, policy: runtime.execPolicy(checkMs) }
+    );
+    if (changed.length > 0) result.reason = `${PROTECTED_FILES_CHANGED} ${changed.join(", ")}.`;
+    else if (!check.ok) result.reason = `The check failed:
+${check.output}`;
+    else result.passed = true;
+  }
+  result.durationMs = Date.now() - started;
+  if (!options.keep) {
+    await rm2(root, { recursive: true, force: true });
+  }
+  return result;
+}
+async function expectedFiles(task, host) {
+  if (task.repo === void 0) return Object.keys(task.solution);
+  if (task.repo.sha === void 0) return void 0;
+  const r = await git(
+    host,
+    task.repo.root,
+    ["diff", "--name-only", task.repo.base, task.repo.sha],
+    {
+      check: false
+    }
+  );
+  if (r.exitCode !== 0) return void 0;
+  const tests = new Set(task.repo.tests.map((t) => t.path));
+  return r.stdout.split("\n").map((line) => line.trim()).filter((line) => line !== "" && !tests.has(line));
+}
+function isAuthFailure(result) {
+  if (!isErrorRun(result) || result.reason === void 0) return false;
+  return /\b40[13]\b|authentication_error|permission_error|invalid x-api-key|invalid api key/i.test(
+    result.reason
+  );
+}
+function isConnectionFailure(result) {
+  if (!isErrorRun(result) || result.reason === void 0) return false;
+  return /ECONNREFUSED|ENOTFOUND|EAI_AGAIN|connection refused|self[- ]signed certificate|unable to get local issuer certificate|certificate has expired/i.test(
+    result.reason
+  );
+}
+async function saveUnique(dir, id, ext, write) {
+  for (let n = 1; ; n++) {
+    const path = join3(dir, `${id}${n === 1 ? "" : `-${n}`}${ext}`);
+    try {
+      await write(path);
+      return path;
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+    }
+  }
+}
+var PROTECTED_FILES_CHANGED = "The agent changed protected files:";
+var MCP_PREPARE_FAILED = "The MCP prepare command failed:";
+var MCP_NOT_STARTED = "The MCP server did not start:";
+function isMcpFailure(result) {
+  return result.reason !== void 0 && (result.reason.startsWith(MCP_PREPARE_FAILED) || result.reason.startsWith(MCP_NOT_STARTED));
+}
+function isSetupFailure(result) {
+  return isAuthFailure(result) || isConnectionFailure(result) || isMcpFailure(result);
+}
+var FAILURE_KINDS = [
+  "check",
+  "protected-files",
+  "step-limit",
+  "token-limit",
+  "repeated-calls",
+  "refusal",
+  "timeout",
+  "error",
+  "setup"
+];
+function failureKind(result) {
+  if (result.passed) return void 0;
+  if (isErrorRun(result)) return isSetupFailure(result) ? "setup" : "error";
+  switch (result.stopReason) {
+    case "timeout":
+      return "timeout";
+    case "max_steps":
+      return "step-limit";
+    case "token_budget":
+    case "max_tokens":
+      return "token-limit";
+    case "repeated_calls":
+      return "repeated-calls";
+    case "refusal":
+      return "refusal";
+  }
+  if (result.reason?.startsWith(PROTECTED_FILES_CHANGED) === true) return "protected-files";
+  return "check";
+}
+function failureSummary(results) {
+  const runs = [];
+  for (const r of results) {
+    const kind = failureKind(r);
+    if (kind === void 0) continue;
+    runs.push({
+      id: r.id,
+      attempt: r.attempt ?? 1,
+      kind,
+      steps: r.steps,
+      tokens: r.tokens,
+      durationMs: r.durationMs
+    });
+  }
+  const byKind = {};
+  for (const kind of FAILURE_KINDS) {
+    const n = runs.filter((f) => f.kind === kind).length;
+    if (n > 0) byKind[kind] = n;
+  }
+  return { total: runs.length, byKind, runs };
+}
+function failureLine(results) {
+  const s = failureSummary(results);
+  if (s.total === 0) return void 0;
+  const parts = FAILURE_KINDS.flatMap((kind) => {
+    const runs = s.runs.filter((f) => f.kind === kind);
+    if (runs.length === 0) return [];
+    const named = runs.slice(0, 3).map((f) => `${f.id} run ${f.attempt}`);
+    if (runs.length > 3) named.push(`${runs.length - 3} more`);
+    return [`${kind} ${runs.length} (${named.join(", ")})`];
+  });
+  return `Failures by kind: ${parts.join(" \xB7 ")}`;
+}
+async function runEvals(tasks, options, onResult, stop) {
+  const results = new Array(tasks.length);
+  const seen = /* @__PURE__ */ new Map();
+  const attempts = tasks.map((t) => {
+    const n = (seen.get(t.id) ?? 0) + 1;
+    seen.set(t.id, n);
+    return n;
+  });
+  let next = 0;
+  let stopped = false;
+  const worker = async () => {
+    for (let i = next++; i < tasks.length && !stopped; i = next++) {
+      const result = await runEvalTask(tasks[i], options);
+      result.attempt = attempts[i];
+      results[i] = result;
+      onResult?.(result);
+      if (stop?.(result)) stopped = true;
+    }
+  };
+  const width = Math.max(1, Math.min(options.parallel ?? 1, tasks.length));
+  await Promise.all(Array.from({ length: width }, worker));
+  return results.filter((r) => r !== void 0);
+}
+async function runCheck(root, command, timeoutMs = 12e4, sandboxed) {
+  const envAllowlist = [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV];
+  const r = sandboxed === void 0 ? await new HostExecutor().run(command, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist,
+    timeoutMs,
+    maxOutputBytes: 4e3
+  }) : await sandboxed.executor.run(command, {
+    ...sandboxed.policy,
+    envAllowlist: [.../* @__PURE__ */ new Set([...sandboxed.policy.envAllowlist, ...TOOLCHAIN_ENV])],
+    timeoutMs,
+    maxOutputBytes: 4e3
+  });
+  const text = `${r.stdout.text}${r.stderr.text}`.trim();
+  const output = text !== "" ? text : `(no output; ${r.timedOut ? "timed out" : `exit code ${r.exitCode}`})`;
+  return { ok: r.exitCode === 0 && !r.timedOut, output };
+}
+async function snapshot(root, paths) {
+  const out = /* @__PURE__ */ new Map();
+  for (const path of paths) out.set(path, await readFile2(join3(root, path), "utf8").catch(() => ""));
+  return out;
+}
+async function changedFiles(root, before) {
+  const after = await snapshot(root, [...before.keys()]);
+  return [...before.keys()].filter((path) => before.get(path) !== after.get(path));
+}
+function isErrorRun(result) {
+  return result.stopReason === "error";
+}
+function summarize(results) {
+  const attempted = results.length;
+  const errors = results.filter(isErrorRun).length;
+  const scored = attempted - errors;
+  const passed = results.filter((r) => r.passed).length;
+  return {
+    attempted,
+    errors,
+    scored,
+    passed,
+    ...scored === 0 ? {} : { scoredRate: passed / scored },
+    ...attempted === 0 ? {} : { attemptedRate: passed / attempted }
+  };
+}
+var percent = (rate) => rate === void 0 ? "\u2013" : `${Math.round(rate * 100)}%`;
+function formatReport(results, options = {}) {
+  const width = idWidth(results);
+  const rows = results.map(
+    (r) => [
+      r.passed ? "PASS" : isErrorRun(r) ? "ERR " : "FAIL",
+      r.id.padEnd(width),
+      `${String(r.steps).padStart(3)} steps`,
+      (r.unreportedTokens ?? 0) > 0 ? `tok ? (${r.unreportedTokens} reserved)` : `${(r.tokens / 1e3).toFixed(1).padStart(7)}k tok`,
+      r.costUsd === void 0 ? "   cost ?" : `${r.costIncomplete === true ? "\u2265" : ""}$${r.costUsd.toFixed(4).padStart(7)}`,
+      `${(r.durationMs / 1e3).toFixed(0).padStart(4)} s`,
+      r.stopReason
+    ].join("  ")
+  );
+  const summary = summarize(results);
+  const known = `$${results.reduce((s, r) => s + (r.costUsd ?? 0), 0).toFixed(4)}`;
+  const cost = !results.every((r) => r.costUsd !== void 0) ? "unknown" : results.some((r) => r.costIncomplete === true) ? `\u2265 ${known} (incomplete)` : known;
+  const steps = results.reduce((s, r) => s + r.steps, 0);
+  const counted = `${summary.passed}/${summary.scored} scored runs passed (${percent(summary.scoredRate)}) \xB7 ${summary.passed}/${summary.attempted} of all attempted runs (${percent(summary.attemptedRate)})` + (summary.errors > 0 ? ` \xB7 ${summary.errors} error run(s)` : "");
+  const tokens = results.reduce((s, r) => s + r.tokens, 0);
+  const cached = results.reduce((s, r) => s + (r.cacheReadTokens ?? 0), 0);
+  const share = results.some((r) => (r.unreportedTokens ?? 0) > 0) ? " \xB7 cache share unknown" : tokens === 0 ? "" : ` \xB7 ${Math.round(cached / tokens * 100)}% of tokens from the cache`;
+  const lines = [
+    ...rows,
+    "",
+    `${counted} \xB7 ${steps} steps \xB7 cost ${cost}${share}`,
+    ...evidenceLines(results, options.seed ?? DEFAULT_SEED)
+  ];
+  const ids = [...new Set(results.map((r) => r.id))];
+  if (ids.length < results.length) lines.push("", "Mean per task:", ...meanRows(results, ids));
+  return lines.join("\n");
+}
+function evidenceLines(results, seed) {
+  const out = [];
+  if (results.length > 0 && results.every(isErrorRun)) {
+    return [
+      "No scored run: every run failed with an error (the reasons are above), so this run gives no evidence about the agent."
+    ];
+  }
+  const interval = passInterval(results, { seed });
+  if (interval !== void 0 && interval.tasks >= 2) {
+    out.push(
+      `Pass rate over all attempts: ${percent(interval.rate)} (95% interval ${percent(interval.low)}\u2013${percent(interval.high)}; cluster bootstrap over ${interval.tasks} tasks, ${interval.draws} draws, seed ${interval.seed})`
+    );
+    if (interval.low === interval.high) {
+      out.push(
+        "  Every draw gave the same rate, so the interval shows no uncertainty: more tasks are needed to measure it."
+      );
+    }
+    if (interval.tasks < 10) out.push(`  ${roughNote(interval.tasks)}`);
+  }
+  const c = consistency(results);
+  if (c.repeated > 0) {
+    const none = c.nonePassed.length === 0 ? "" : ` (${c.nonePassed.join(", ")})`;
+    const errors = c.onlyErrors.length === 0 ? "" : `; ${c.onlyErrors.length} had only error runs (${c.onlyErrors.join(", ")})`;
+    out.push(
+      `Consistency: ${c.allPassed.length} of ${c.repeated} repeated tasks passed every run; ${c.nonePassed.length} passed none${none}${errors}`
+    );
+  }
+  const s = spread(results);
+  if (s !== void 0) {
+    const k = (n) => `${(n / 1e3).toFixed(1)}k`;
+    const sec = (ms) => `${Math.round(ms / 1e3)} s`;
+    out.push(
+      `Spread (${s.runs} scored runs): steps median ${s.steps.median.toFixed(1)}, p95 ${s.steps.p95.toFixed(1)} \xB7 tokens median ${k(s.tokens.median)}, p95 ${k(s.tokens.p95)} \xB7 time median ${sec(s.durationMs.median)}, p95 ${sec(s.durationMs.p95)}`
+    );
+  }
+  const cost = costPerPass(results);
+  out.push(
+    cost.kind === "unknown" ? "Cost per accepted change: unknown (a run's cost is unknown)" : cost.kind === "none" ? "Cost per accepted change: no passes" : `Cost per accepted change: ${cost.minimum ? "\u2265 " : ""}$${cost.usd.toFixed(4)} (all attempts ${cost.minimum ? "\u2265 " : ""}$${cost.totalUsd.toFixed(4)} / ${cost.passes} pass${cost.passes === 1 ? "" : "es"})`
+  );
+  const u = unexpectedChanges(results);
+  if (u.measured > 0) {
+    out.push(
+      `Unexpected changes: ${u.runs} of ${u.measured} runs, ${u.files} file${u.files === 1 ? "" : "s"}`
+    );
+    const shown = results.filter((r) => (r.unexpected?.length ?? 0) > 0).slice(0, 5);
+    for (const r of shown) {
+      out.push(
+        `  ${r.id}${r.attempt === void 0 ? "" : ` (run ${r.attempt})`}: ${(r.unexpected ?? []).join(", ")}`
+      );
+    }
+    if (u.runs > shown.length) out.push(`  \u2026 ${u.runs - shown.length} more in report.json`);
+  }
+  const mcp = mcpUse(results);
+  if (mcp !== void 0) out.push(`MCP tool calls: ${mcpUseText(mcp)}`);
+  const failures = failureLine(results);
+  if (failures !== void 0) out.push(failures);
+  return out;
+}
+function idWidth(results) {
+  return Math.max(18, ...results.map((r) => r.id.length));
+}
+function meanRows(results, ids) {
+  const width = idWidth(results);
+  return ids.map((id) => {
+    const all = results.filter((r) => r.id === id);
+    const runs = all.filter((r) => !isErrorRun(r));
+    const errors = all.length - runs.length;
+    const passesAll = all.filter((r) => r.passed).length;
+    const note = errors > 0 ? `  (${passesAll}/${all.length} of all attempts; ${errors} error run(s))` : "";
+    if (runs.length === 0) return `${"0/0".padEnd(4)}  ${id.padEnd(width)}${note}`;
+    const mean = (f) => runs.reduce((s, r) => s + f(r), 0) / runs.length;
+    const passes = runs.filter((r) => r.passed).length;
+    return [
+      `${passes}/${runs.length}`.padEnd(4),
+      id.padEnd(width),
+      `${mean((r) => r.steps).toFixed(1).padStart(5)} steps`,
+      runs.some((r) => (r.unreportedTokens ?? 0) > 0) ? "    tok ?" : `${(mean((r) => r.tokens) / 1e3).toFixed(1).padStart(7)}k tok`,
+      runs.every((r) => r.costUsd !== void 0) ? `${runs.some((r) => r.costIncomplete === true) ? "\u2265" : ""}$${mean((r) => r.costUsd ?? 0).toFixed(4).padStart(7)}` : "   cost ?",
+      ...stepsSd(runs) === void 0 ? [] : [`sd ${stepsSd(runs).toFixed(1)} steps`]
+    ].join("  ") + note;
+  });
+}
+var EVAL_BIOME_CONFIG = {
+  formatter: { indentStyle: "space", indentWidth: 2, lineWidth: 100 },
+  linter: { enabled: false },
+  assist: { enabled: false }
+};
+async function formatProject(root, biome) {
+  await writeFile2(join3(root, "biome.json"), `${JSON.stringify(EVAL_BIOME_CONFIG, null, 2)}
+`);
+  const result = await new HostExecutor().run(`${shellWord(biome)} format --write .`, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: [...DEFAULT_ENV_ALLOWLIST],
+    timeoutMs: 6e4,
+    maxOutputBytes: 2e4
+  });
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `Biome could not format the eval project: ${result.stderr.text.trim().split("\n")[0] ?? ""}`
+    );
+  }
+}
+function shellWord(text) {
+  return /^[A-Za-z0-9_./:@=+-]+$/.test(text) ? text : `'${text.replace(/'/g, "'\\''")}'`;
+}
+
+// src/evals/report.ts
+import { createHash as createHash2 } from "crypto";
+import { arch, platform, release } from "os";
+var REPORT_SCHEMA = 2;
+function taskHash(task) {
+  const sorted = (record) => Object.fromEntries(Object.entries(record).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
+  const data = {
+    prompt: task.prompt,
+    files: sorted(task.files),
+    check: task.check,
+    protect: task.protect ?? null,
+    solution: sorted(task.solution),
+    allowChanges: task.allowChanges ?? null,
+    repo: task.repo === void 0 ? null : { base: task.repo.base, sha: task.repo.sha ?? null, tests: task.repo.tests }
+  };
+  return createHash2("sha256").update(JSON.stringify(data)).digest("hex");
+}
+function buildFields(build, checkout) {
+  const commit = build ?? checkout;
+  return {
+    version: VERSION,
+    ...commit === void 0 ? {} : { commit },
+    ...build !== void 0 && checkout !== void 0 && checkout !== build ? { checkout } : {}
+  };
+}
+async function reportV2(meta, tasks, results) {
+  return {
+    schema: REPORT_SCHEMA,
+    garuda: buildFields(BUILD_COMMIT, await garudaCommit()),
+    spec: meta.model,
+    agent: meta.agent ?? { name: "garuda" },
+    provider: meta.provider,
+    // Evals set no effort: each model's default applies.
+    effort: "model default",
+    isolation: meta.isolation,
+    os: { platform: platform(), release: release(), arch: arch(), node: process.versions.node },
+    seed: meta.seed,
+    shuffle: meta.shuffle,
+    split: meta.split ?? "dev",
+    addendum: meta.addendum === void 0 ? null : {
+      file: meta.addendum.file,
+      chars: meta.addendum.text.length,
+      sha256: createHash2("sha256").update(meta.addendum.text).digest("hex")
+    },
+    mcp: meta.mcp === void 0 ? null : {
+      server: meta.mcp.server,
+      ...meta.mcp.prepare === void 0 ? {} : { prepare: meta.mcp.prepare }
+    },
+    order: results.map((r) => ({ id: r.id, attempt: r.attempt ?? 1 })),
+    tasks: Object.fromEntries(tasks.map((t) => [t.id, taskHash(t)])),
+    evidence: {
+      interval: passInterval(results, { seed: meta.seed }) ?? null,
+      consistency: consistency(results),
+      spread: spread(results) ?? null,
+      stepsSd: Object.fromEntries(byTask(results).map((t) => [t.id, stepsSd(t.results) ?? null])),
+      costPerAcceptedChange: costPerPass(results),
+      unexpected: unexpectedChanges(results),
+      failures: failureSummary(results)
+    }
+  };
+}
+
+export {
+  isAuthFailure,
+  isMcpFailure,
+  isSetupFailure,
+  runEvals,
+  summarize,
+  formatReport,
+  taskHash,
+  reportV2
+};
+//# sourceMappingURL=chunk-DUYLVGPY.js.map