@garuda-agent/garuda 0.0.1 → 0.15.0

package/dist/evalCommand-3SSFMZQD.js

@@ -0,0 +1,2798 @@
+#!/usr/bin/env node
+import {
+  AutoApprover
+} from "./chunk-NXJ4XLB5.js";
+import {
+  loadModelsConfig,
+  resolveModel
+} from "./chunk-YFQJXRIJ.js";
+import {
+  CODE_INDEX_MODES,
+  DEFAULT_CODE_INDEX_MODE,
+  FileSessionStore,
+  Runtime,
+  detectProfiles,
+  newSessionId
+} from "./chunk-K7KK7U44.js";
+import "./chunk-A2QAIRQL.js";
+import {
+  totalTokens
+} from "./chunk-HRGX53VC.js";
+import "./chunk-G2AX7MGS.js";
+import "./chunk-B25VL4RU.js";
+import "./chunk-23FXBNBB.js";
+import "./chunk-EPKWDCV4.js";
+import "./chunk-5DEWP3YX.js";
+import "./chunk-ZAE25JM2.js";
+import "./chunk-ATW7BNIG.js";
+import "./chunk-PZS3V6P3.js";
+import {
+  LINK_CANDIDATES,
+  cleanArgs
+} from "./chunk-K3SIUOAG.js";
+import {
+  MANIFEST,
+  TEST_PATH,
+  detectTestCommand,
+  tail
+} from "./chunk-OUCV3FCT.js";
+import {
+  git,
+  shellWord
+} from "./chunk-2TAXSFWN.js";
+import {
+  DEFAULT_ENV_ALLOWLIST,
+  EXECUTOR_NAMES,
+  HostExecutor,
+  createExecutor,
+  expandAllowlist,
+  parseSettings
+} from "./chunk-SI2QJYCR.js";
+import "./chunk-WMOLXTBU.js";
+import "./chunk-HUIK3WXO.js";
+import "./chunk-3NJNVZTK.js";
+
+// src/cli/evalCommand.ts
+import { existsSync as existsSync2, mkdirSync as mkdirSync3, realpathSync as realpathSync3, writeFileSync } from "fs";
+import { createRequire } from "module";
+import { dirname as dirname3, join as join5 } from "path";
+
+// src/evals/repoTasks.ts
+import { mkdtempSync, realpathSync } from "fs";
+import { mkdir, readFile, rm as rm2, symlink, writeFile as writeFile2 } from "fs/promises";
+import { tmpdir as tmpdir2 } from "os";
+import { dirname as dirname2, join as join3 } from "path";
+import { z } from "zod";
+
+// src/evals/toolchains.ts
+import { mkdtemp, rm } from "fs/promises";
+import { tmpdir } from "os";
+import { join as join2 } from "path";
+
+// src/evals/files.ts
+import { mkdirSync } from "fs";
+import { writeFile } from "fs/promises";
+import { dirname, join } from "path";
+async function writeFiles(root, files) {
+  for (const [path, content] of Object.entries(files)) {
+    const full = join(root, path);
+    mkdirSync(dirname(full), { recursive: true });
+    await writeFile(full, content);
+  }
+}
+
+// src/evals/projects.ts
+var MAVEN_POM = `<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>dev.garuda.eval</groupId>
+  <artifactId>eval</artifactId>
+  <version>1.0.0</version>
+
+  <properties>
+    <maven.compiler.release>17</maven.compiler.release>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.11.4</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <version>3.3.1</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.13.0</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.5.2</version>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+`;
+var MAVEN_CHECK = "test ! -e .mvn && mvn -B -q -o test";
+var PYPROJECT = `[project]
+name = "evalpkg"
+version = "0.1.0"
+requires-python = ">=3.9"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+`;
+var PYTEST_CHECK = "python3 -m pytest -q -c pyproject.toml --noconftest -p no:cacheprovider";
+function isProtectedPath(path) {
+  return /^(test|tests|src\/test)\//.test(path) || path === "pom.xml" || path === "pyproject.toml";
+}
+var lines = (...l) => `${l.join("\n")}
+`;
+
+// src/evals/toolchains.ts
+var TOOLCHAIN_ENV = ["JAVA_HOME", "GRADLE_USER_HOME", "VIRTUAL_ENV"];
+var TOOLCHAINS = {
+  maven: {
+    id: "maven",
+    title: "Maven with JUnit 5 in ~/.m2",
+    files: {
+      "pom.xml": MAVEN_POM,
+      "src/main/java/probe/Probe.java": "package probe;\n\npublic final class Probe {\n  public static int one() {\n    return 1;\n  }\n}\n",
+      "src/test/java/probe/ProbeTest.java": "package probe;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport org.junit.jupiter.api.Test;\n\nclass ProbeTest {\n  @Test\n  void one() {\n    assertEquals(1, Probe.one());\n  }\n}\n"
+    },
+    probe: "mvn -B -q -o test",
+    prepare: "mvn -B -q test",
+    hint: "The Java tasks need a JDK (17 or later), Maven, and JUnit 5 in ~/.m2. Run `garuda eval --prepare java` once: it downloads the plugins and JUnit with Maven (network)."
+  },
+  pytest: {
+    id: "pytest",
+    title: "Python 3 with pytest",
+    files: {
+      "pyproject.toml": PYPROJECT,
+      "src/probe/__init__.py": "def one():\n    return 1\n",
+      "tests/test_probe.py": "from probe import one\n\n\ndef test_one():\n    assert one() == 1\n"
+    },
+    probe: "python3 -m pytest -q -p no:cacheprovider",
+    hint: "The Python tasks need python3 with pytest 7 or later. Install it, for example with `python3 -m pip install --user pytest`."
+  }
+};
+var PROBE_TIMEOUT_MS = 12e4;
+var PREPARE_TIMEOUT_MS = 10 * 6e4;
+async function checkToolchains(ids) {
+  return Promise.all(
+    [...new Set(ids)].map(async (id) => {
+      const toolchain = TOOLCHAINS[id];
+      const r = await inScratch(toolchain, toolchain.probe, PROBE_TIMEOUT_MS);
+      return { id, ...r };
+    })
+  );
+}
+async function prepareToolchain(id) {
+  const toolchain = TOOLCHAINS[id];
+  if (toolchain.prepare === void 0) return { ok: false, output: toolchain.hint };
+  return inScratch(toolchain, toolchain.prepare, PREPARE_TIMEOUT_MS);
+}
+function toolchainId(name) {
+  const aliases = {
+    java: "maven",
+    maven: "maven",
+    python: "pytest",
+    pytest: "pytest"
+  };
+  return aliases[name.toLowerCase()];
+}
+async function inScratch(toolchain, command, timeoutMs) {
+  const root = await mkdtemp(join2(tmpdir(), `garuda-toolchain-${toolchain.id}-`));
+  try {
+    await writeFiles(root, toolchain.files);
+    const r = await new HostExecutor().run(command, {
+      root,
+      sandbox: false,
+      writePaths: [root],
+      denyWritePaths: [],
+      denyReadPaths: [],
+      network: true,
+      envAllowlist: [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV],
+      timeoutMs,
+      maxOutputBytes: 4e3
+    });
+    const output = `${r.stdout.text}${r.stderr.text}`.trim();
+    return { ok: r.exitCode === 0 && !r.timedOut, output };
+  } catch (error) {
+    return { ok: false, output: error.message };
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+// src/evals/repoTasks.ts
+var REPO_SUITE_FILE = join3(".garuda", "evals", "repo-suite.json");
+var DEFAULT_COMMITS = 200;
+var DEFAULT_MAX_TASKS = 30;
+var MAX_TASK_FILES = 10;
+var BUILD_TEST_TIMEOUT_MS = 5 * 6e4;
+var testFile = z.object({ path: z.string(), content: z.string().nullable() });
+var taskSchema = z.object({
+  id: z.string(),
+  sha: z.string(),
+  base: z.string(),
+  subject: z.string(),
+  prompt: z.string(),
+  tests: z.array(testFile),
+  /** The check: the test command for this task's test files. Absent: the suite's test command. */
+  check: z.string().optional()
+});
+var suiteSchema = z.object({
+  version: z.literal(1),
+  root: z.string(),
+  head: z.string(),
+  createdAt: z.string(),
+  testCommand: z.string(),
+  tasks: z.array(taskSchema)
+});
+var DOC_PATH = /\.(md|mdx|rst|txt|adoc)$|(^|\/)docs?\//i;
+var PACKAGE_JSON = /(^|\/)package\.json$/;
+function skipReason(commit) {
+  const { files } = commit;
+  if (files.length === 0) return "no files";
+  const code = files.filter((f) => !DOC_PATH.test(f.path));
+  if (code.length > MAX_TASK_FILES) return `more than ${MAX_TASK_FILES} files`;
+  if (files.some((f) => f.added === void 0)) return "a binary file";
+  if (files.some((f) => MANIFEST.test(f.path) && !PACKAGE_JSON.test(f.path))) {
+    return "dependencies changed";
+  }
+  const tests = code.filter((f) => TEST_PATH.test(f.path));
+  if (tests.length === 0) return "no test change";
+  if (tests.length === code.length) return "only tests changed";
+  return void 0;
+}
+function dependencyChange(diff) {
+  for (const line of diff.split("\n")) {
+    if (!/^[+-]/.test(line) || /^(\+\+\+|---)/.test(line)) continue;
+    if (/[dD]ependencies"/.test(line)) return true;
+    const entry = /^[+-]\s*"([^"]+)":\s*"[~^<>=]*\d/.exec(line);
+    if (entry !== null && entry[1] !== "version") return true;
+  }
+  return false;
+}
+function scopedTestCommand(command, files) {
+  const list = files.map(shellWord).join(" ");
+  if (command.includes("{files}")) return command.replaceAll("{files}", list);
+  if (files.length === 0) return command;
+  const c = command.trim();
+  if (/^npm (run )?test$/.test(c)) return `${c} -- ${list}`;
+  if (/^(pnpm|yarn)( run)? test$/.test(c) || /^node --test$/.test(c) || /^(npx |pnpm exec )?(vitest|jest)( run)?$/.test(c) || /-m pytest( -q)?$|^pytest( -q)?$/.test(c)) {
+    return `${c} ${list}`;
+  }
+  return c;
+}
+function repoPrompt(commit, tests) {
+  return [
+    commit.subject.trim(),
+    ...commit.body.trim() === "" ? [] : ["", commit.body.trim()],
+    "",
+    `The tests for this change are already in the project: ${tests.join(", ")}. Make the change so that they pass. Do not change these test files.`
+  ].join("\n");
+}
+async function recentCommits(executor, root, options = {}) {
+  const log = await git(executor, root, [
+    "log",
+    "--no-merges",
+    "--format=%H%x00%s%x00%b%x1e",
+    `-n${options.commits ?? DEFAULT_COMMITS}`,
+    ...options.since === void 0 ? [] : [`--since=${options.since}`],
+    "HEAD"
+  ]);
+  const out = [];
+  for (const entry of log.stdout.split("")) {
+    const [sha = "", subject = "", body = ""] = entry.replace(/^\n/, "").split("\0");
+    if (!/^[0-9a-f]{40,64}$/.test(sha)) continue;
+    const stat = await git(executor, root, ["show", "--no-renames", "--numstat", "--format=", sha]);
+    const files = stat.stdout.split("\n").map((line) => /^(\d+|-)\t(\d+|-)\t(.+)$/.exec(line)).filter((m) => m !== null).map(
+      (m) => m[1] === "-" ? { path: m[3] } : { path: m[3], added: Number(m[1]), removed: Number(m[2]) }
+    );
+    out.push({ sha, subject, body, files });
+  }
+  return out;
+}
+async function taskWorktree(executor, root, commit) {
+  const dir = realpathSync(mkdtempSync(join3(tmpdir2(), "garuda-repo-task-")));
+  await git(executor, root, ["worktree", "add", "--detach", "--quiet", dir, commit]);
+  for (const name of LINK_CANDIDATES) {
+    try {
+      await symlink(join3(root, name), join3(dir, name), "dir");
+    } catch {
+    }
+  }
+  return dir;
+}
+async function removeTaskWorktree(executor, root, dir) {
+  await git(executor, root, ["worktree", "remove", "--force", dir], { check: false });
+  await rm2(dir, { recursive: true, force: true });
+  await git(executor, root, ["worktree", "prune"], { check: false });
+}
+async function applyTests(dir, tests) {
+  for (const t of tests) {
+    const file = join3(dir, t.path);
+    if (t.content === null) await rm2(file, { force: true });
+    else {
+      await mkdir(dirname2(file), { recursive: true });
+      await writeFile2(file, t.content);
+    }
+  }
+}
+async function runTests(executor, dir, command) {
+  const result = await executor.run(command, {
+    root: dir,
+    sandbox: false,
+    writePaths: [dir],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV],
+    timeoutMs: BUILD_TEST_TIMEOUT_MS,
+    maxOutputBytes: 2e4
+  });
+  const output = `${result.stdout.text}${result.stderr.text}`;
+  return {
+    ok: result.exitCode === 0 && !result.timedOut,
+    tail: result.timedOut ? "(stopped: too long)" : tail(output, 15)
+  };
+}
+async function buildRepoSuite(executor, root, options) {
+  const say = options.onProgress ?? (() => {
+  });
+  const head = (await git(executor, root, ["rev-parse", "HEAD"])).stdout.trim();
+  const commits = await recentCommits(executor, root, {
+    ...options.commits === void 0 ? {} : { commits: options.commits },
+    ...options.since === void 0 ? {} : { since: options.since }
+  });
+  const skipped = {};
+  const skip = (why) => {
+    skipped[why] = (skipped[why] ?? 0) + 1;
+  };
+  const tasks = [];
+  let checked = 0;
+  let failure;
+  const max = options.maxTasks ?? DEFAULT_MAX_TASKS;
+  for (const commit of commits) {
+    if (tasks.length >= max) break;
+    const why = skipReason(commit);
+    if (why !== void 0) {
+      skip(why);
+      continue;
+    }
+    const parent = await git(
+      executor,
+      root,
+      ["rev-parse", "--verify", "--quiet", `${commit.sha}^`],
+      {
+        check: false
+      }
+    );
+    if (parent.exitCode !== 0) {
+      skip("the first commit (no parent)");
+      continue;
+    }
+    if (commit.files.some((f) => PACKAGE_JSON.test(f.path))) {
+      const diff = await git(
+        executor,
+        root,
+        ["show", "--format=", commit.sha, "--", "package.json"],
+        {
+          check: false
+        }
+      );
+      if (dependencyChange(diff.stdout)) {
+        skip("dependencies changed");
+        continue;
+      }
+    }
+    const base2 = parent.stdout.trim();
+    const testPaths = commit.files.filter((f) => TEST_PATH.test(f.path) && !DOC_PATH.test(f.path)).map((f) => f.path);
+    const tests = [];
+    for (const path of testPaths) {
+      const shown = await git(executor, root, ["show", `${commit.sha}:${path}`], { check: false });
+      tests.push({ path, content: shown.exitCode === 0 ? shown.stdout : null });
+    }
+    const runnable = tests.filter((t) => t.content !== null).map((t) => t.path);
+    if (runnable.length === 0) {
+      skip("only deletes tests");
+      continue;
+    }
+    const check = scopedTestCommand(options.testCommand, runnable);
+    checked++;
+    say(`checking ${commit.sha.slice(0, 7)} ${commit.subject}`);
+    const dir = await taskWorktree(executor, root, base2);
+    try {
+      await applyTests(dir, tests);
+      if ((await runTests(executor, dir, check)).ok) {
+        skip("the tests already pass at the parent");
+        continue;
+      }
+      await git(executor, dir, ["checkout", "--quiet", "--", "."], { check: false });
+      await git(executor, dir, cleanArgs(), { check: false });
+      await git(executor, dir, ["checkout", "--quiet", "--detach", commit.sha]);
+      const atCommit = await runTests(executor, dir, check);
+      if (!atCommit.ok) {
+        skip("the tests fail at the commit");
+        failure ??= { sha: commit.sha, command: check, tail: atCommit.tail };
+        continue;
+      }
+    } finally {
+      await removeTaskWorktree(executor, root, dir);
+    }
+    tasks.push({
+      id: `repo-${commit.sha.slice(0, 7)}`,
+      sha: commit.sha,
+      base: base2,
+      subject: commit.subject,
+      prompt: repoPrompt(commit, testPaths),
+      tests,
+      check
+    });
+    say(`kept ${commit.sha.slice(0, 7)} (${tasks.length}/${max})`);
+  }
+  return {
+    suite: {
+      version: 1,
+      root,
+      head,
+      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+      testCommand: options.testCommand,
+      tasks
+    },
+    checked,
+    skipped,
+    ...failure === void 0 ? {} : { failure }
+  };
+}
+async function saveRepoSuite(suite) {
+  const file = join3(suite.root, REPO_SUITE_FILE);
+  await mkdir(dirname2(file), { recursive: true });
+  await writeFile2(file, `${JSON.stringify(suite, null, 2)}
+`, { mode: 384 });
+  return file;
+}
+async function loadRepoSuite(root) {
+  let text;
+  try {
+    text = await readFile(join3(root, REPO_SUITE_FILE), "utf8");
+  } catch {
+    return void 0;
+  }
+  const parsed = suiteSchema.safeParse(JSON.parse(text));
+  if (!parsed.success) throw new Error(`${REPO_SUITE_FILE}: ${z.prettifyError(parsed.error)}`);
+  return parsed.data;
+}
+function repoEvalTasks(suite) {
+  return suite.tasks.map((t) => ({
+    id: t.id,
+    title: t.subject,
+    prompt: t.prompt,
+    files: {},
+    check: t.check ?? suite.testCommand,
+    protect: t.tests.filter((f) => f.content !== null).map((f) => f.path),
+    solution: {},
+    repo: { root: suite.root, base: t.base, tests: t.tests }
+  }));
+}
+
+// src/evals/runner.ts
+import { existsSync, mkdirSync as mkdirSync2, realpathSync as realpathSync2 } from "fs";
+import { copyFile, mkdtemp as mkdtemp2, readFile as readFile2, rm as rm3, writeFile as writeFile3 } from "fs/promises";
+import { tmpdir as tmpdir3 } from "os";
+import { join as join4 } from "path";
+var EVAL_DENY_RULES = [
+  "bash(rm -rf*)",
+  "bash(sudo*)",
+  "bash(git push*)",
+  "bash(curl*)",
+  "bash(wget*)"
+];
+var REPO_CHECK_TIMEOUT_MS = 5 * 6e4;
+var EVAL_TASK_TIMEOUT_MS = 10 * 6e4;
+async function runEvalTask(task4, options) {
+  const started = Date.now();
+  const host = new HostExecutor();
+  let root;
+  if (task4.repo === void 0) {
+    root = realpathSync2(await mkdtemp2(join4(tmpdir3(), `garuda-eval-${task4.id}-`)));
+    await writeFiles(root, task4.files);
+  } else {
+    root = await taskWorktree(host, task4.repo.root, task4.repo.base);
+    await applyTests(root, task4.repo.tests);
+  }
+  if (options.format !== void 0) await formatProject(root, options.format.biome);
+  const protect = task4.protect ?? Object.keys(task4.files).filter(isProtectedPath);
+  const before = await snapshot(root, protect);
+  const settings = parseSettings({
+    // With an allowlist, a host outside it is denied: nobody answers questions in an eval.
+    permissions: {
+      deny: options.network === void 0 ? EVAL_DENY_RULES : [...EVAL_DENY_RULES, "network"]
+    },
+    ...options.network === void 0 ? {} : { network: { allow: options.network } },
+    // Evals must not depend on the internet.
+    web: { enabled: false },
+    ...options.executor === void 0 ? {} : { executor: options.executor },
+    ...options.codeIndex === void 0 ? {} : { codeIndex: options.codeIndex },
+    ...options.maxSteps === void 0 ? {} : { limits: { maxSteps: options.maxSteps } },
+    ...options.subagents === void 0 ? {} : { subagents: { enabled: options.subagents } },
+    ...options.todo === void 0 ? {} : { todo: { enabled: options.todo } },
+    ...options.format === void 0 ? {} : {
+      formatters: {
+        enabled: options.format.on,
+        commands: {
+          biome: {
+            extensions: ["js", "jsx", "mjs", "cjs", "ts", "tsx", "json"],
+            command: [options.format.biome, "format", "--write", "$FILE"]
+          }
+        }
+      }
+    },
+    ...options.keepThinking === void 0 ? {} : { thinking: { keepBlocks: options.keepThinking } }
+  });
+  const store = new FileSessionStore(root);
+  const runtime = await Runtime.create({
+    root,
+    modelId: options.modelId,
+    model: async () => options.model(),
+    ...options.modelInfo === void 0 ? {} : { modelInfo: options.modelInfo },
+    ...options.maxTokens === void 0 ? {} : { maxTokens: options.maxTokens },
+    ...options.subagentModel === void 0 ? {} : { subagentModel: options.subagentModel },
+    approver: new AutoApprover("once"),
+    store,
+    settings,
+    // The user's own MCP servers must not change eval results.
+    mcp: false,
+    hooks: false,
+    ...options.policy === void 0 ? {} : { policy: options.policy },
+    commands: false,
+    // The list comes from the eval command line: the user typed it.
+    ...options.network === void 0 ? {} : { network: { approved: true } },
+    ...options.lsp === void 0 ? {} : { lsp: { enabled: options.lsp } },
+    ...options.onEvent === void 0 ? {} : { onEvent: (event) => options.onEvent?.(task4.id, event) }
+  });
+  const result = {
+    id: task4.id,
+    title: task4.title,
+    passed: false,
+    stopReason: "error",
+    steps: 0,
+    tokens: 0,
+    costUsd: void 0,
+    durationMs: 0
+  };
+  const limitMs = options.taskTimeoutMs ?? EVAL_TASK_TIMEOUT_MS;
+  const timeout = AbortSignal.timeout(limitMs);
+  try {
+    const run = await runtime.runTurn(task4.prompt, timeout);
+    result.stopReason = run.stopReason;
+    result.steps = run.steps;
+  } catch (error) {
+    runtime.recordStop("error");
+    if (timeout.aborted) {
+      result.stopReason = "timeout";
+      result.reason = `The task took longer than ${limitMs / 6e4} minutes.`;
+    } else {
+      result.reason = `The run failed: ${error.message}`;
+    }
+  } finally {
+    await runtime.close();
+    runtime.executor.shutdown();
+  }
+  const session = runtime.session;
+  if (session !== void 0) {
+    result.tokens = totalTokens(session.usage);
+    result.costUsd = session.costUsd;
+    result.cacheReadTokens = session.usage.cacheReadTokens;
+    if (options.outDir !== void 0) {
+      mkdirSync2(options.outDir, { recursive: true });
+      let target = join4(options.outDir, `${task4.id}.jsonl`);
+      for (let n = 2; existsSync(target); n++)
+        target = join4(options.outDir, `${task4.id}-${n}.jsonl`);
+      await copyFile(store.path(session.id), target);
+      result.sessionFile = target;
+    }
+  }
+  if (result.reason === void 0) {
+    const changed = await changedFiles(root, before);
+    const checkMs = task4.repo === void 0 ? 12e4 : REPO_CHECK_TIMEOUT_MS;
+    const check = await runCheck(
+      root,
+      task4.check,
+      checkMs,
+      runtime.executor.isolation === "none" ? void 0 : { executor: runtime.executor, policy: runtime.execPolicy(checkMs) }
+    );
+    if (changed.length > 0)
+      result.reason = `The agent changed protected files: ${changed.join(", ")}.`;
+    else if (!check.ok) result.reason = `The check failed:
+${check.output}`;
+    else result.passed = true;
+  }
+  result.durationMs = Date.now() - started;
+  if (!options.keep) {
+    if (task4.repo === void 0) await rm3(root, { recursive: true, force: true });
+    else await removeTaskWorktree(host, task4.repo.root, root);
+  }
+  return result;
+}
+async function runEvals(tasks, options, onResult) {
+  const results = new Array(tasks.length);
+  let next = 0;
+  const worker = async () => {
+    for (let i = next++; i < tasks.length; i = next++) {
+      const result = await runEvalTask(tasks[i], options);
+      results[i] = result;
+      onResult?.(result);
+    }
+  };
+  const width = Math.max(1, Math.min(options.parallel ?? 1, tasks.length));
+  await Promise.all(Array.from({ length: width }, worker));
+  return results;
+}
+async function runCheck(root, command, timeoutMs = 12e4, sandboxed) {
+  const envAllowlist = [...DEFAULT_ENV_ALLOWLIST, ...TOOLCHAIN_ENV];
+  const r = sandboxed === void 0 ? await new HostExecutor().run(command, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist,
+    timeoutMs,
+    maxOutputBytes: 4e3
+  }) : await sandboxed.executor.run(command, {
+    ...sandboxed.policy,
+    envAllowlist: [.../* @__PURE__ */ new Set([...sandboxed.policy.envAllowlist, ...TOOLCHAIN_ENV])],
+    timeoutMs,
+    maxOutputBytes: 4e3
+  });
+  const output = `${r.stdout.text}${r.stderr.text}`.trim();
+  return { ok: r.exitCode === 0 && !r.timedOut, output };
+}
+async function snapshot(root, paths) {
+  const out = /* @__PURE__ */ new Map();
+  for (const path of paths) out.set(path, await readFile2(join4(root, path), "utf8").catch(() => ""));
+  return out;
+}
+async function changedFiles(root, before) {
+  const after = await snapshot(root, [...before.keys()]);
+  return [...before.keys()].filter((path) => before.get(path) !== after.get(path));
+}
+function isErrorRun(result) {
+  return result.stopReason === "error";
+}
+function formatReport(results) {
+  const rows = results.map(
+    (r) => [
+      r.passed ? "PASS" : isErrorRun(r) ? "ERR " : "FAIL",
+      r.id.padEnd(18),
+      `${String(r.steps).padStart(3)} steps`,
+      `${(r.tokens / 1e3).toFixed(1).padStart(7)}k tok`,
+      r.costUsd === void 0 ? "   cost ?" : `$${r.costUsd.toFixed(4).padStart(7)}`,
+      `${(r.durationMs / 1e3).toFixed(0).padStart(4)} s`,
+      r.stopReason
+    ].join("  ")
+  );
+  const passed = results.filter((r) => r.passed).length;
+  const errors = results.filter(isErrorRun).length;
+  const cost = results.every((r) => r.costUsd !== void 0) ? `$${results.reduce((s, r) => s + (r.costUsd ?? 0), 0).toFixed(4)}` : "unknown";
+  const steps = results.reduce((s, r) => s + r.steps, 0);
+  const counted = `${passed}/${results.length - errors} passed${errors > 0 ? ` \xB7 ${errors} error run(s) not counted` : ""}`;
+  const tokens = results.reduce((s, r) => s + r.tokens, 0);
+  const cached = results.reduce((s, r) => s + (r.cacheReadTokens ?? 0), 0);
+  const share = tokens === 0 ? "" : ` \xB7 ${Math.round(cached / tokens * 100)}% of tokens from the cache`;
+  const lines4 = [...rows, "", `${counted} \xB7 ${steps} steps \xB7 cost ${cost}${share}`];
+  const ids = [...new Set(results.map((r) => r.id))];
+  if (ids.length < results.length) lines4.push("", "Mean per task:", ...meanRows(results, ids));
+  return lines4.join("\n");
+}
+function meanRows(results, ids) {
+  return ids.map((id) => {
+    const all = results.filter((r) => r.id === id);
+    const runs = all.filter((r) => !isErrorRun(r));
+    const errors = all.length - runs.length;
+    const note = errors > 0 ? `  (${errors} error run(s) left out)` : "";
+    if (runs.length === 0) return `${"0/0".padEnd(4)}  ${id.padEnd(18)}${note}`;
+    const mean = (f) => runs.reduce((s, r) => s + f(r), 0) / runs.length;
+    const passes = runs.filter((r) => r.passed).length;
+    return [
+      `${passes}/${runs.length}`.padEnd(4),
+      id.padEnd(18),
+      `${mean((r) => r.steps).toFixed(1).padStart(5)} steps`,
+      `${(mean((r) => r.tokens) / 1e3).toFixed(1).padStart(7)}k tok`,
+      runs.every((r) => r.costUsd !== void 0) ? `$${mean((r) => r.costUsd ?? 0).toFixed(4).padStart(7)}` : "   cost ?"
+    ].join("  ") + note;
+  });
+}
+var EVAL_BIOME_CONFIG = {
+  formatter: { indentStyle: "space", indentWidth: 2, lineWidth: 100 },
+  linter: { enabled: false },
+  assist: { enabled: false }
+};
+async function formatProject(root, biome) {
+  await writeFile3(join4(root, "biome.json"), `${JSON.stringify(EVAL_BIOME_CONFIG, null, 2)}
+`);
+  const result = await new HostExecutor().run(`${shellWord2(biome)} format --write .`, {
+    root,
+    sandbox: false,
+    writePaths: [root],
+    denyWritePaths: [],
+    denyReadPaths: [],
+    network: false,
+    envAllowlist: [...DEFAULT_ENV_ALLOWLIST],
+    timeoutMs: 6e4,
+    maxOutputBytes: 2e4
+  });
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `Biome could not format the eval project: ${result.stderr.text.trim().split("\n")[0] ?? ""}`
+    );
+  }
+}
+function shellWord2(text) {
+  return /^[A-Za-z0-9_./:@=+-]+$/.test(text) ? text : `'${text.replace(/'/g, "'\\''")}'`;
+}
+
+// src/evals/shopkit.ts
+var lines2 = (...l) => `${l.join("\n")}
+`;
+var entity = (name, prefix, fields, amount) => ({
+  name,
+  Name: name[0]?.toUpperCase() + name.slice(1),
+  prefix,
+  fields,
+  ...amount === void 0 ? {} : { amount }
+});
+var ENTITIES = [
+  entity("product", "prd", ["name", "price"], "price"),
+  entity("customer", "cus", ["name", "email"]),
+  entity("cart", "crt", ["customerId"]),
+  entity("shipment", "shp", ["orderId", "carrier"]),
+  entity("warehouse", "whs", ["name", "city"]),
+  entity("supplier", "sup", ["name", "country"]),
+  entity("coupon", "cpn", ["code", "percent"]),
+  entity("review", "rev", ["productId", "stars"]),
+  entity("category", "cat", ["name"]),
+  entity("payment", "pay", ["orderId", "amount"], "amount"),
+  entity("refund", "ref", ["paymentId", "amount"], "amount"),
+  entity("address", "adr", ["customerId", "line1", "postcode"]),
+  entity("price", "prc", ["productId", "value"]),
+  entity("tax", "tax", ["region", "rate"]),
+  entity("discount", "dsc", ["name", "amount"], "amount"),
+  entity("report", "rpt", ["title"]),
+  entity("user", "usr", ["login", "role"])
+];
+function model(e) {
+  return lines2(
+    'import { newId } from "../../core/ids.js";',
+    "",
+    `/** Create a ${e.name} record. */`,
+    `export function create${e.Name}(fields) {`,
+    `  return { id: newId("${e.prefix}"), ...fields };`,
+    "}"
+  );
+}
+function validate(e) {
+  return lines2(
+    'import { ValidationError } from "../../core/errors.js";',
+    "",
+    `const REQUIRED = [${e.fields.map((f) => `"${f}"`).join(", ")}];`,
+    "",
+    `export function validate${e.Name}(input) {`,
+    "  for (const key of REQUIRED) {",
+    `    if (input?.[key] === undefined) throw new ValidationError(\`${e.name}.\${key} is required\`);`,
+    "  }",
+    "  return input;",
+    "}"
+  );
+}
+function repository(e) {
+  return lines2(
+    `export class ${e.Name}Repository {`,
+    "  #items = new Map();",
+    "",
+    "  add(item) {",
+    "    this.#items.set(item.id, item);",
+    "    return item;",
+    "  }",
+    "",
+    "  get(id) {",
+    "    return this.#items.get(id);",
+    "  }",
+    "",
+    "  listAll() {",
+    "    return [...this.#items.values()];",
+    "  }",
+    "",
+    "  remove(id) {",
+    "    return this.#items.delete(id);",
+    "  }",
+    "}"
+  );
+}
+function service(e) {
+  const money2 = e.amount !== void 0;
+  return lines2(
+    'import { events } from "../../core/events.js";',
+    ...money2 ? ['import { formatPrice, toCents } from "../../core/money.js";'] : [],
+    `import { create${e.Name} } from "./model.js";`,
+    `import { ${e.Name}Repository } from "./repository.js";`,
+    `import { validate${e.Name} } from "./validate.js";`,
+    "",
+    `export class ${e.Name}Service {`,
+    `  constructor({ repo = new ${e.Name}Repository(), bus = events } = {}) {`,
+    "    this.repo = repo;",
+    "    this.bus = bus;",
+    "  }",
+    "",
+    "  create(input) {",
+    `    const item = this.repo.add(create${e.Name}(validate${e.Name}(input)));`,
+    `    this.bus.emit("${e.name}.created", item);`,
+    "    return item;",
+    "  }",
+    "",
+    "  find(id) {",
+    "    return this.repo.get(id);",
+    "  }",
+    "",
+    "  all() {",
+    "    return this.repo.listAll();",
+    "  }",
+    ...money2 ? [
+      "",
+      "  describe(item) {",
+      `    return \`${e.Name} \${item.id}: \${formatPrice(toCents(item.${e.amount}))}\`;`,
+      "  }"
+    ] : [],
+    "}"
+  );
+}
+function entityTest(e) {
+  const sample = Object.fromEntries(e.fields.map((f, i) => [f, i === 0 ? `x-${e.name}` : 3]));
+  return lines2(
+    'import assert from "node:assert/strict";',
+    'import { test } from "node:test";',
+    'import { EventBus } from "../../src/core/events.js";',
+    `import { ${e.Name}Service } from "../../src/domains/${e.name}/service.js";`,
+    "",
+    `test("${e.name}: create, find, list", () => {`,
+    `  const service = new ${e.Name}Service({ bus: new EventBus() });`,
+    `  const item = service.create(${JSON.stringify(sample)});`,
+    "  assert.equal(service.find(item.id), item);",
+    "  assert.equal(service.all().length, 1);",
+    `  assert.match(item.id, /^${e.prefix}_/);`,
+    "});",
+    "",
+    `test("${e.name}: validation", () => {`,
+    `  const service = new ${e.Name}Service({ bus: new EventBus() });`,
+    `  assert.throws(() => service.create({}), /${e.name}\\.${e.fields[0]} is required/);`,
+    "});"
+  );
+}
+var CORE = {
+  "src/core/ids.js": lines2(
+    "let next = 1;",
+    "",
+    "/** Ids like prd_000001. Sequential, so tests are stable. */",
+    "export function newId(prefix) {",
+    '  return `${prefix}_${String(next++).padStart(6, "0")}`;',
+    "}"
+  ),
+  "src/core/errors.js": lines2(
+    "export class ValidationError extends Error {",
+    '  name = "ValidationError";',
+    "}",
+    "",
+    "export class NotFoundError extends Error {",
+    '  name = "NotFoundError";',
+    "}"
+  ),
+  "src/core/events.js": lines2(
+    "export class EventBus {",
+    "  #handlers = new Map();",
+    "",
+    "  on(name, handler) {",
+    "    const list = this.#handlers.get(name) ?? [];",
+    "    list.push(handler);",
+    "    this.#handlers.set(name, list);",
+    "  }",
+    "",
+    "  emit(name, payload) {",
+    "    for (const handler of this.#handlers.get(name) ?? []) handler(payload);",
+    "  }",
+    "}",
+    "",
+    "/** The shared bus. Tests pass their own. */",
+    "export const events = new EventBus();"
+  ),
+  "src/core/money.js": lines2(
+    "/** Amounts are numbers in pounds. Totals are integers in pence. */",
+    "export function toCents(amount) {",
+    "  return Math.round(amount * 100);",
+    "}",
+    "",
+    'export function formatPrice(cents, symbol = "\xA3") {',
+    '  const sign = cents < 0 ? "-" : "";',
+    "  const abs = Math.abs(cents);",
+    '  return `${sign}${symbol}${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}`;',
+    "}"
+  ),
+  "src/core/strings.js": lines2(
+    "export function capitalize(text) {",
+    "  return text.charAt(0).toUpperCase() + text.slice(1);",
+    "}",
+    "",
+    "export function slug(text) {",
+    '  return text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");',
+    "}",
+    "",
+    "export function truncate(text, max) {",
+    "  return text.length <= max ? text : `${text.slice(0, max - 1)}\u2026`;",
+    "}",
+    "",
+    'export function padLeft(text, width, fill = " ") {',
+    "  return String(text).padStart(width, fill);",
+    "}"
+  ),
+  "src/core/dates.js": lines2(
+    "const DAY = 24 * 60 * 60 * 1000;",
+    "",
+    "export function addDays(date, days) {",
+    "  return new Date(date.getTime() + days * DAY);",
+    "}",
+    "",
+    "export function toIsoDate(date) {",
+    "  return date.toISOString().slice(0, 10);",
+    "}",
+    "",
+    "export function isWeekend(date) {",
+    "  const day = date.getUTCDay();",
+    "  return day === 0 || day === 6;",
+    "}"
+  ),
+  "src/core/config.js": lines2(
+    'const DEFAULTS = { port: 3000, currency: "GBP", invoiceDays: 30 };',
+    "",
+    "export function loadConfig(overrides = {}) {",
+    "  return { ...DEFAULTS, ...overrides };",
+    "}"
+  ),
+  "src/core/logger.js": lines2(
+    "export function createLogger(name, sink = () => {}) {",
+    "  return {",
+    "    info: (message) => sink(`[${name}] ${message}`),",
+    "    error: (message) => sink(`[${name}] ERROR ${message}`),",
+    "  };",
+    "}"
+  )
+};
+var HAND_WRITTEN = {
+  // order: totals, events, describe
+  "src/domains/order/model.js": model(entity("order", "ord", [])),
+  "src/domains/order/repository.js": repository(entity("order", "ord", [])),
+  "src/domains/order/service.js": lines2(
+    'import { NotFoundError, ValidationError } from "../../core/errors.js";',
+    'import { events } from "../../core/events.js";',
+    'import { formatPrice, toCents } from "../../core/money.js";',
+    'import { createOrder } from "./model.js";',
+    'import { OrderRepository } from "./repository.js";',
+    "",
+    "export class OrderService {",
+    "  constructor({ products, repo = new OrderRepository(), bus = events } = {}) {",
+    "    this.products = products;",
+    "    this.repo = repo;",
+    "    this.bus = bus;",
+    "  }",
+    "",
+    "  create({ customerId, lines }) {",
+    '    if (!lines?.length) throw new ValidationError("order.lines is required");',
+    "    let totalCents = 0;",
+    "    for (const line of lines) {",
+    "      const product = this.products.find(line.productId);",
+    "      if (!product) throw new NotFoundError(`product ${line.productId}`);",
+    "      totalCents += toCents(product.price) * line.qty;",
+    "    }",
+    "    const order = this.repo.add(createOrder({ customerId, lines, totalCents }));",
+    '    this.bus.emit("order.created", order);',
+    "    return order;",
+    "  }",
+    "",
+    "  find(id) {",
+    "    return this.repo.get(id);",
+    "  }",
+    "",
+    "  all() {",
+    "    return this.repo.listAll();",
+    "  }",
+    "",
+    "  describe(order) {",
+    "    return `Order ${order.id}: ${formatPrice(order.totalCents)}`;",
+    "  }",
+    "}"
+  ),
+  // stock: reacts to order.created
+  "src/domains/stock/service.js": lines2(
+    'import { events } from "../../core/events.js";',
+    "",
+    "/** Stock levels per product. Orders reserve stock through the order.created event. */",
+    "export class StockService {",
+    "  #levels = new Map();",
+    "",
+    "  constructor({ bus = events } = {}) {",
+    '    bus.on("order.created", (order) => this.reserve(order));',
+    "  }",
+    "",
+    "  set(productId, qty) {",
+    "    this.#levels.set(productId, qty);",
+    "  }",
+    "",
+    "  level(productId) {",
+    "    return this.#levels.get(productId) ?? 0;",
+    "  }",
+    "",
+    "  reserve(order) {",
+    "    for (const line of order.lines) this.set(line.productId, this.level(line.productId) - line.qty);",
+    "  }",
+    "}"
+  ),
+  // invoice: formats the order total
+  "src/domains/invoice/service.js": lines2(
+    'import { loadConfig } from "../../core/config.js";',
+    'import { addDays, toIsoDate } from "../../core/dates.js";',
+    'import { formatPrice } from "../../core/money.js";',
+    "",
+    "export class InvoiceService {",
+    "  constructor({ config = loadConfig() } = {}) {",
+    "    this.config = config;",
+    "  }",
+    "",
+    "  fromOrder(order, issued = new Date(Date.UTC(2026, 0, 1))) {",
+    "    return {",
+    "      orderId: order.id,",
+    "      amount: formatPrice(order.totalCents),",
+    "      issuedOn: toIsoDate(issued),",
+    "      dueOn: toIsoDate(addDays(issued, this.config.invoiceDays)),",
+    "    };",
+    "  }",
+    "}"
+  ),
+  "src/api/routes.js": lines2(
+    'import { loadConfig } from "../core/config.js";',
+    'import { createLogger } from "../core/logger.js";',
+    'import { capitalize, slug } from "../core/strings.js";',
+    'import { CustomerService } from "../domains/customer/service.js";',
+    'import { OrderService } from "../domains/order/service.js";',
+    'import { ProductService } from "../domains/product/service.js";',
+    'import { ShipmentService } from "../domains/shipment/service.js";',
+    "",
+    "/** Build the route table. Handlers take a body and return a result. */",
+    'export function buildRoutes(log = createLogger("api")) {',
+    "  const config = loadConfig();",
+    "  const products = new ProductService();",
+    "  const orders = new OrderService({ products });",
+    "  const customers = new CustomerService();",
+    "  const shipments = new ShipmentService();",
+    "  const routes = {",
+    '    "POST /products": (body) => products.create(body),',
+    '    "POST /customers": (body) => customers.create(body),',
+    '    "POST /orders": (body) => orders.create(body),',
+    '    "POST /shipments": (body) => shipments.create(body),',
+    "  };",
+    "  log.info(`${Object.keys(routes).length} routes on port ${config.port}`);",
+    "  return routes;",
+    "}",
+    "",
+    "export function routeTitle(route) {",
+    '  return capitalize(slug(route.split(" ")[1] ?? ""));',
+    "}"
+  )
+};
+var imp = (names, from) => [
+  'import assert from "node:assert/strict";',
+  'import { test } from "node:test";',
+  `import { ${names} } from "${from}";`
+].join("\n");
+var FEATURE_TESTS = {
+  "test/features/order-total.test.js": lines2(
+    imp("EventBus", "../../src/core/events.js"),
+    'import { OrderService } from "../../src/domains/order/service.js";',
+    'import { ProductService } from "../../src/domains/product/service.js";',
+    "",
+    'test("order total in pence", () => {',
+    "  const bus = new EventBus();",
+    "  const products = new ProductService({ bus });",
+    '  const pen = products.create({ name: "pen", price: 0.29 });',
+    '  const pad = products.create({ name: "pad", price: 1.15 });',
+    "  const orders = new OrderService({ products, bus });",
+    '  const order = orders.create({ customerId: "c1", lines: [{ productId: pen.id, qty: 3 }, { productId: pad.id, qty: 1 }] });',
+    "  assert.equal(order.totalCents, 202);",
+    "  assert.equal(orders.describe(order), `Order ${order.id}: \xA32.02`);",
+    "});"
+  ),
+  "test/features/invoice.test.js": lines2(
+    imp("EventBus", "../../src/core/events.js"),
+    'import { InvoiceService } from "../../src/domains/invoice/service.js";',
+    'import { OrderService } from "../../src/domains/order/service.js";',
+    'import { ProductService } from "../../src/domains/product/service.js";',
+    "",
+    'test("invoice from an order", () => {',
+    "  const bus = new EventBus();",
+    "  const products = new ProductService({ bus });",
+    '  const tea = products.create({ name: "tea", price: 0.57 });',
+    '  const order = new OrderService({ products, bus }).create({ customerId: "c1", lines: [{ productId: tea.id, qty: 1 }] });',
+    "  const invoice = new InvoiceService().fromOrder(order);",
+    '  assert.deepEqual(invoice, { orderId: order.id, amount: "\xA30.57", issuedOn: "2026-01-01", dueOn: "2026-01-31" });',
+    "});"
+  ),
+  "test/features/stock.test.js": lines2(
+    imp("EventBus", "../../src/core/events.js"),
+    'import { OrderService } from "../../src/domains/order/service.js";',
+    'import { ProductService } from "../../src/domains/product/service.js";',
+    'import { StockService } from "../../src/domains/stock/service.js";',
+    "",
+    'test("an order reserves stock", () => {',
+    "  const bus = new EventBus();",
+    "  const products = new ProductService({ bus });",
+    "  const stock = new StockService({ bus });",
+    '  const mug = products.create({ name: "mug", price: 4 });',
+    "  stock.set(mug.id, 10);",
+    '  new OrderService({ products, bus }).create({ customerId: "c1", lines: [{ productId: mug.id, qty: 3 }] });',
+    "  assert.equal(stock.level(mug.id), 7);",
+    "});"
+  ),
+  "test/api/routes.test.js": lines2(
+    imp("buildRoutes, routeTitle", "../../src/api/routes.js"),
+    "",
+    'test("routes", () => {',
+    "  const logs = [];",
+    "  const routes = buildRoutes({ info: (m) => logs.push(m), error: () => {} });",
+    '  assert.deepEqual(Object.keys(routes).sort(), ["POST /customers", "POST /orders", "POST /products", "POST /shipments"]);',
+    '  assert.deepEqual(logs, ["4 routes on port 3000"]);',
+    '  assert.equal(routeTitle("POST /products"), "Products");',
+    "});"
+  ),
+  "test/core/money.test.js": lines2(
+    imp("formatPrice, toCents", "../../src/core/money.js"),
+    "",
+    'test("money", () => {',
+    "  assert.equal(toCents(0.29), 29);",
+    '  assert.equal(formatPrice(1205), "\xA312.05");',
+    '  assert.equal(formatPrice(-5), "-\xA30.05");',
+    "});"
+  ),
+  "test/core/strings.test.js": lines2(
+    imp("capitalize, padLeft, slug, truncate", "../../src/core/strings.js"),
+    "",
+    'test("strings", () => {',
+    '  assert.equal(capitalize("abc"), "Abc");',
+    '  assert.equal(slug("Hello World!"), "hello-world");',
+    '  assert.equal(truncate("abcdef", 4), "abc\u2026");',
+    '  assert.equal(padLeft(7, 3, "0"), "007");',
+    "});"
+  )
+};
+function shopkit() {
+  const files = {
+    "package.json": '{ "name": "shopkit", "version": "2.4.0", "type": "module", "private": true }\n',
+    "README.md": lines2(
+      "# shopkit",
+      "",
+      "A small shop backend. Domains live in src/domains/<name>/ (model, repository, service, validate).",
+      "Shared code lives in src/core/. The route table is in src/api/routes.js.",
+      "Run the tests with: node --test"
+    ),
+    ...CORE,
+    ...HAND_WRITTEN,
+    ...FEATURE_TESTS
+  };
+  for (const e of ENTITIES) {
+    files[`src/domains/${e.name}/model.js`] = model(e);
+    files[`src/domains/${e.name}/validate.js`] = validate(e);
+    files[`src/domains/${e.name}/repository.js`] = repository(e);
+    files[`src/domains/${e.name}/service.js`] = service(e);
+    files[`test/domains/${e.name}.test.js`] = entityTest(e);
+  }
+  return files;
+}
+
+// src/evals/hardTasks.ts
+var BASE = shopkit();
+var base = (path) => {
+  const content = BASE[path];
+  if (content === void 0) throw new Error(`shopkit has no ${path}`);
+  return content;
+};
+function task(fields) {
+  const { change, ...rest } = fields;
+  return { ...rest, files: { ...BASE, ...change } };
+}
+var replaceIn = (path, from, to) => {
+  const content = base(path);
+  if (!content.includes(from)) throw new Error(`${path} has no "${from}"`);
+  return content.split(from).join(to);
+};
+var renamed = Object.fromEntries(
+  Object.entries(BASE).filter(([path, content]) => path.startsWith("src/") && content.includes("formatPrice")).map(([path, content]) => [path, content.split("formatPrice").join("formatMoney")])
+);
+var HARD_TASKS = [
+  task({
+    id: "hard-rounding",
+    title: "Find a root cause in shared code",
+    prompt: "Several tests about orders and invoices fail. Find the root cause and fix it. Do not change the tests.",
+    change: {
+      "src/core/money.js": replaceIn(
+        "src/core/money.js",
+        "Math.round(amount * 100)",
+        "Math.floor(amount * 100)"
+      )
+    },
+    check: "node --test",
+    solution: { "src/core/money.js": base("src/core/money.js") }
+  }),
+  task({
+    id: "hard-event",
+    title: "Fix a mismatch between two modules",
+    prompt: "Creating an order no longer reserves stock. Find out why and fix it. Do not change the tests.",
+    change: {
+      "src/domains/stock/service.js": replaceIn(
+        "src/domains/stock/service.js",
+        '"order.created"',
+        '"orders.created"'
+      )
+    },
+    check: "node --test",
+    solution: { "src/domains/stock/service.js": base("src/domains/stock/service.js") }
+  }),
+  task({
+    id: "hard-import",
+    title: "Fix a broken import in a large repo",
+    prompt: "The API routes test crashes. Fix it. Do not change the tests.",
+    change: {
+      "src/api/routes.js": replaceIn(
+        "src/api/routes.js",
+        "domains/shipment/service.js",
+        "domains/shipments/service.js"
+      )
+    },
+    check: "node --test",
+    solution: { "src/api/routes.js": base("src/api/routes.js") }
+  }),
+  task({
+    id: "hard-rename",
+    title: "Rename a shared function in every file",
+    prompt: "Rename the function formatPrice (in src/core/money.js) to formatMoney: the definition and every import and call in src/. The tests already use the new name.",
+    change: {
+      "test/core/money.test.js": base("test/core/money.test.js").split("formatPrice").join("formatMoney")
+    },
+    check: "node --test && ! grep -rq formatPrice src",
+    solution: renamed
+  }),
+  task({
+    id: "hard-unused",
+    title: "Find unused exports",
+    prompt: "Which exported functions in src/core/ are not used by any other file in src/? Count only functions declared with the `function` keyword: classes and constants do not count. Uses in test/ do not count. Write the names to unused.txt, one per line, in alphabetical order. Do not change any other file.",
+    change: {},
+    // diff prints the expected and the actual lines, so a failure shows what the agent wrote.
+    check: 'diff <(printf "isWeekend\\npadLeft\\ntruncate\\n") <(tr -d " \\r" < unused.txt | grep -v "^$")',
+    protect: Object.keys(BASE).filter((p) => p.startsWith("src/")),
+    solution: { "unused.txt": "isWeekend\npadLeft\ntruncate\n" }
+  }),
+  task({
+    id: "hard-coupon",
+    title: "Add a feature across modules",
+    prompt: "Add coupon support to orders. When OrderService.create() gets a couponCode, find the coupon with that code and take its percent off the total (round to whole pence). An unknown code must throw a NotFoundError. test/features/coupon.test.js describes the behavior. Do not change the tests.",
+    change: {
+      "test/features/coupon.test.js": [
+        'import assert from "node:assert/strict";',
+        'import { test } from "node:test";',
+        'import { NotFoundError } from "../../src/core/errors.js";',
+        'import { EventBus } from "../../src/core/events.js";',
+        'import { CouponService } from "../../src/domains/coupon/service.js";',
+        'import { OrderService } from "../../src/domains/order/service.js";',
+        'import { ProductService } from "../../src/domains/product/service.js";',
+        "",
+        "function setup() {",
+        "  const bus = new EventBus();",
+        "  const products = new ProductService({ bus });",
+        "  const coupons = new CouponService({ bus });",
+        '  coupons.create({ code: "SAVE10", percent: 10 });',
+        '  const lamp = products.create({ name: "lamp", price: 19.99 });',
+        "  return { orders: new OrderService({ products, coupons, bus }), lamp };",
+        "}",
+        "",
+        'test("a coupon takes its percent off", () => {',
+        "  const { orders, lamp } = setup();",
+        '  const order = orders.create({ customerId: "c1", couponCode: "SAVE10", lines: [{ productId: lamp.id, qty: 2 }] });',
+        "  assert.equal(order.totalCents, 3598);",
+        "});",
+        "",
+        'test("no coupon, full price", () => {',
+        "  const { orders, lamp } = setup();",
+        '  assert.equal(orders.create({ customerId: "c1", lines: [{ productId: lamp.id, qty: 2 }] }).totalCents, 3998);',
+        "});",
+        "",
+        'test("an unknown code throws", () => {',
+        "  const { orders, lamp } = setup();",
+        "  assert.throws(",
+        '    () => orders.create({ customerId: "c1", couponCode: "NOPE", lines: [{ productId: lamp.id, qty: 1 }] }),',
+        "    NotFoundError,",
+        "  );",
+        "});",
+        ""
+      ].join("\n")
+    },
+    check: "node --test",
+    solution: {
+      "src/domains/order/service.js": base("src/domains/order/service.js").replace(
+        "constructor({ products, repo = new OrderRepository(), bus = events } = {}) {\n    this.products = products;",
+        "constructor({ products, coupons, repo = new OrderRepository(), bus = events } = {}) {\n    this.products = products;\n    this.coupons = coupons;"
+      ).replace("create({ customerId, lines }) {", "create({ customerId, lines, couponCode }) {").replace(
+        "    const order = this.repo.add(",
+        [
+          "    if (couponCode !== undefined) {",
+          "      const coupon = this.coupons?.all().find((c) => c.code === couponCode);",
+          "      if (!coupon) throw new NotFoundError(`coupon ${couponCode}`);",
+          "      totalCents = Math.round(totalCents * (1 - coupon.percent / 100));",
+          "    }",
+          "    const order = this.repo.add("
+        ].join("\n")
+      )
+    }
+  })
+];
+
+// src/evals/javaTasks.ts
+var java = (pkg2, ...body) => lines(`package ${pkg2};`, "", ...body);
+var junit = (pkg2, ...body) => java(
+  pkg2,
+  "import static org.junit.jupiter.api.Assertions.*;",
+  "",
+  "import org.junit.jupiter.api.Test;",
+  ...body
+);
+var main = (path) => `src/main/java/${path}`;
+var test = (path) => `src/test/java/${path}`;
+var task2 = (t) => ({
+  ...t,
+  files: { "pom.xml": MAVEN_POM, ...t.files },
+  check: MAVEN_CHECK,
+  requires: ["maven"]
+});
+var paginator = (end) => java(
+  "app",
+  "import java.util.List;",
+  "",
+  "public final class Paginator {",
+  "  private Paginator() {}",
+  "",
+  "  /** Page numbers start at 1. A page after the last one is empty. */",
+  "  public static <T> List<T> page(List<T> items, int page, int size) {",
+  "    if (page < 1 || size < 1) {",
+  '      throw new IllegalArgumentException("page and size must be positive");',
+  "    }",
+  "    int from = (page - 1) * size;",
+  "    if (from >= items.size()) {",
+  "      return List.of();",
+  "    }",
+  `    int to = ${end};`,
+  "    return items.subList(from, to);",
+  "  }",
+  "}"
+);
+var money = (...extra) => java(
+  "app",
+  "import java.math.BigDecimal;",
+  ...extra.length > 0 ? ["import java.util.Objects;"] : [],
+  "",
+  "/** An amount of money. 1.5 GBP and 1.50 GBP are equal. */",
+  "public final class Money {",
+  "  private final BigDecimal amount;",
+  "  private final String currency;",
+  "",
+  "  public Money(String amount, String currency) {",
+  "    this.amount = new BigDecimal(amount);",
+  "    this.currency = currency;",
+  "  }",
+  "",
+  "  @Override",
+  "  public boolean equals(Object other) {",
+  "    if (this == other) {",
+  "      return true;",
+  "    }",
+  "    if (!(other instanceof Money m)) {",
+  "      return false;",
+  "    }",
+  "    return amount.compareTo(m.amount) == 0 && currency.equals(m.currency);",
+  "  }",
+  ...extra,
+  "",
+  "  @Override",
+  "  public String toString() {",
+  '    return amount.toPlainString() + " " + currency;',
+  "  }",
+  "}"
+);
+var repo = (name) => java(
+  "app",
+  "import java.util.Map;",
+  "",
+  "public final class UserRepo {",
+  '  private final Map<Integer, String> names = Map.of(1, "ada", 2, "linus");',
+  "",
+  `  public String ${name}(int id) {`,
+  '    return names.getOrDefault(id, "unknown");',
+  "  }",
+  "}"
+);
+var profile = (name) => java(
+  "app",
+  "public final class ProfileService {",
+  "  private final UserRepo repo;",
+  "",
+  "  public ProfileService(UserRepo repo) {",
+  "    this.repo = repo;",
+  "  }",
+  "",
+  "  public String title(int id) {",
+  `    return "Profile of " + repo.${name}(id);`,
+  "  }",
+  "}"
+);
+var admin = (name) => java(
+  "app",
+  "public final class AdminService {",
+  "  private final UserRepo repo;",
+  "",
+  "  public AdminService(UserRepo repo) {",
+  "    this.repo = repo;",
+  "  }",
+  "",
+  "  public boolean isAdmin(int id) {",
+  `    return "ada".equals(repo.${name}(id));`,
+  "  }",
+  "}"
+);
+var report = (body, imports2) => java(
+  "shop",
+  ...imports2,
+  "",
+  "public final class OrderReport {",
+  "  private OrderReport() {}",
+  "",
+  "  /** Total per customer, in customer name order. Cancelled orders do not count. */",
+  "  public static Map<String, BigDecimal> totalsByCustomer(List<Order> orders) {",
+  ...body,
+  "  }",
+  "}"
+);
+var JAVA_TASKS = [
+  task2({
+    id: "java-paginate",
+    title: "Java: fix the last page of a paginator",
+    prompt: "The tests of this Maven project fail: the last page of Paginator is wrong. Fix the bug. Do not change the tests.",
+    files: {
+      [main("app/Paginator.java")]: paginator("Math.min(from + size, items.size() - 1)"),
+      [test("app/PaginatorTest.java")]: junit(
+        "app",
+        "import java.util.List;",
+        "",
+        "class PaginatorTest {",
+        "  private final List<Integer> items = List.of(1, 2, 3, 4, 5);",
+        "",
+        "  @Test",
+        "  void fullPages() {",
+        "    assertEquals(List.of(1, 2), Paginator.page(items, 1, 2));",
+        "    assertEquals(List.of(3, 4), Paginator.page(items, 2, 2));",
+        "  }",
+        "",
+        "  @Test",
+        "  void lastPageMayBeShort() {",
+        "    assertEquals(List.of(5), Paginator.page(items, 3, 2));",
+        "    assertEquals(items, Paginator.page(items, 1, 10));",
+        "  }",
+        "",
+        "  @Test",
+        "  void pageAfterTheEndIsEmpty() {",
+        "    assertEquals(List.of(), Paginator.page(items, 4, 2));",
+        "  }",
+        "",
+        "  @Test",
+        "  void rejectsBadArguments() {",
+        "    assertThrows(IllegalArgumentException.class, () -> Paginator.page(items, 0, 2));",
+        "  }",
+        "}"
+      )
+    },
+    solution: { [main("app/Paginator.java")]: paginator("Math.min(from + size, items.size())") }
+  }),
+  task2({
+    id: "java-hashcode",
+    title: "Java: make Money work in a HashSet",
+    prompt: "Money objects that are equal do not work as keys in a HashSet or HashMap, and the tests fail. Fix Money. Do not change the tests.",
+    files: {
+      [main("app/Money.java")]: money(),
+      [test("app/MoneyTest.java")]: junit(
+        "app",
+        "import java.util.HashMap;",
+        "import java.util.HashSet;",
+        "import java.util.Map;",
+        "import java.util.Set;",
+        "",
+        "class MoneyTest {",
+        "  @Test",
+        "  void equalAmountsAreEqual() {",
+        '    assertEquals(new Money("1.5", "GBP"), new Money("1.50", "GBP"));',
+        '    assertNotEquals(new Money("1.5", "GBP"), new Money("1.5", "EUR"));',
+        "  }",
+        "",
+        "  @Test",
+        "  void equalMoneyIsOneSetElement() {",
+        "    Set<Money> set = new HashSet<>();",
+        '    set.add(new Money("1.50", "GBP"));',
+        '    set.add(new Money("1.5", "GBP"));',
+        "    assertEquals(1, set.size());",
+        '    assertTrue(set.contains(new Money("1.500", "GBP")));',
+        "  }",
+        "",
+        "  @Test",
+        "  void worksAsAMapKey() {",
+        "    Map<Money, String> prices = new HashMap<>();",
+        '    prices.put(new Money("2", "EUR"), "tea");',
+        '    assertEquals("tea", prices.get(new Money("2.00", "EUR")));',
+        '    assertNull(prices.get(new Money("2", "GBP")));',
+        "  }",
+        "}"
+      )
+    },
+    solution: {
+      [main("app/Money.java")]: money(
+        "",
+        "  @Override",
+        "  public int hashCode() {",
+        "    return Objects.hash(amount.stripTrailingZeros(), currency);",
+        "  }"
+      )
+    }
+  }),
+  task2({
+    id: "java-rename",
+    title: "Java: rename a method in every class",
+    prompt: "Rename the method findUsr of UserRepo to findUser everywhere in src/main/java (the definition and every call). Then run the tests.",
+    files: {
+      [main("app/UserRepo.java")]: repo("findUsr"),
+      [main("app/ProfileService.java")]: profile("findUsr"),
+      [main("app/AdminService.java")]: admin("findUsr"),
+      [test("app/ServicesTest.java")]: junit(
+        "app",
+        "class ServicesTest {",
+        "  private final UserRepo repo = new UserRepo();",
+        "",
+        "  @Test",
+        "  void repoFindsUsers() {",
+        '    assertEquals("ada", repo.findUser(1));',
+        '    assertEquals("unknown", repo.findUser(9));',
+        "  }",
+        "",
+        "  @Test",
+        "  void servicesUseTheRepo() {",
+        '    assertEquals("Profile of linus", new ProfileService(repo).title(2));',
+        "    assertTrue(new AdminService(repo).isAdmin(1));",
+        "  }",
+        "}"
+      )
+    },
+    solution: {
+      [main("app/UserRepo.java")]: repo("findUser"),
+      [main("app/ProfileService.java")]: profile("findUser"),
+      [main("app/AdminService.java")]: admin("findUser")
+    }
+  }),
+  task2({
+    id: "java-lru-cache",
+    title: "Java: implement an LRU cache from its tests",
+    prompt: "Implement LruCache so that its tests pass. Read the tests first. Do not change the tests.",
+    files: {
+      [main("cache/LruCache.java")]: java(
+        "cache",
+        "/** A cache with a fixed capacity. When it is full, put removes the least recently used entry. */",
+        "public final class LruCache<K, V> {",
+        "  public LruCache(int capacity) {",
+        '    throw new UnsupportedOperationException("not implemented");',
+        "  }",
+        "",
+        "  /** The value, or null. A hit makes the entry the most recently used. */",
+        "  public V get(K key) {",
+        '    throw new UnsupportedOperationException("not implemented");',
+        "  }",
+        "",
+        "  public void put(K key, V value) {",
+        '    throw new UnsupportedOperationException("not implemented");',
+        "  }",
+        "",
+        "  public int size() {",
+        '    throw new UnsupportedOperationException("not implemented");',
+        "  }",
+        "}"
+      ),
+      [test("cache/LruCacheTest.java")]: junit(
+        "cache",
+        "class LruCacheTest {",
+        "  @Test",
+        "  void evictsTheLeastRecentlyUsedEntry() {",
+        "    LruCache<String, Integer> cache = new LruCache<>(2);",
+        '    cache.put("a", 1);',
+        '    cache.put("b", 2);',
+        '    cache.put("c", 3);',
+        '    assertNull(cache.get("a"));',
+        '    assertEquals(2, cache.get("b"));',
+        '    assertEquals(3, cache.get("c"));',
+        "    assertEquals(2, cache.size());",
+        "  }",
+        "",
+        "  @Test",
+        "  void getMakesAnEntryRecent() {",
+        "    LruCache<String, Integer> cache = new LruCache<>(2);",
+        '    cache.put("a", 1);',
+        '    cache.put("b", 2);',
+        '    cache.get("a");',
+        '    cache.put("c", 3);',
+        '    assertEquals(1, cache.get("a"));',
+        '    assertNull(cache.get("b"));',
+        "  }",
+        "",
+        "  @Test",
+        "  void putReplacesAValueAndMakesItRecent() {",
+        "    LruCache<String, Integer> cache = new LruCache<>(2);",
+        '    cache.put("a", 1);',
+        '    cache.put("b", 2);',
+        '    cache.put("a", 10);',
+        '    cache.put("c", 3);',
+        '    assertEquals(10, cache.get("a"));',
+        '    assertNull(cache.get("b"));',
+        "    assertEquals(2, cache.size());",
+        "  }",
+        "",
+        "  @Test",
+        "  void rejectsAnEmptyCapacity() {",
+        "    assertThrows(IllegalArgumentException.class, () -> new LruCache<String, Integer>(0));",
+        "  }",
+        "}"
+      )
+    },
+    solution: {
+      [main("cache/LruCache.java")]: java(
+        "cache",
+        "import java.util.LinkedHashMap;",
+        "import java.util.Map;",
+        "",
+        "public final class LruCache<K, V> {",
+        "  private final Map<K, V> map;",
+        "",
+        "  public LruCache(int capacity) {",
+        "    if (capacity < 1) {",
+        '      throw new IllegalArgumentException("capacity must be positive");',
+        "    }",
+        "    this.map = new LinkedHashMap<>(16, 0.75f, true) {",
+        "      @Override",
+        "      protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {",
+        "        return size() > capacity;",
+        "      }",
+        "    };",
+        "  }",
+        "",
+        "  public V get(K key) {",
+        "    return map.get(key);",
+        "  }",
+        "",
+        "  public void put(K key, V value) {",
+        "    map.put(key, value);",
+        "  }",
+        "",
+        "  public int size() {",
+        "    return map.size();",
+        "  }",
+        "}"
+      )
+    }
+  }),
+  task2({
+    id: "java-report",
+    title: "Java: fix a stream report with two bugs",
+    prompt: "OrderReport.totalsByCustomer is wrong: cancelled orders must not count, and customers must be in name order. The tests fail. Fix OrderReport. Do not change the tests or Order.",
+    protect: ["pom.xml", test("shop/OrderReportTest.java"), main("shop/Order.java")],
+    files: {
+      [main("shop/Order.java")]: java(
+        "shop",
+        "import java.math.BigDecimal;",
+        "",
+        "public record Order(String customer, BigDecimal total, Status status) {",
+        "  public enum Status {",
+        "    OPEN,",
+        "    PAID,",
+        "    CANCELLED",
+        "  }",
+        "}"
+      ),
+      [main("shop/OrderReport.java")]: report(
+        [
+          "    Map<String, BigDecimal> totals = new HashMap<>();",
+          "    for (Order order : orders) {",
+          "      totals.merge(order.customer(), order.total(), BigDecimal::add);",
+          "    }",
+          "    return totals;"
+        ],
+        [
+          "import java.math.BigDecimal;",
+          "import java.util.HashMap;",
+          "import java.util.List;",
+          "import java.util.Map;"
+        ]
+      ),
+      [test("shop/OrderReportTest.java")]: junit(
+        "shop",
+        "import java.math.BigDecimal;",
+        "import java.util.ArrayList;",
+        "import java.util.List;",
+        "import java.util.Map;",
+        "",
+        "class OrderReportTest {",
+        "  private static Order order(String customer, String total, Order.Status status) {",
+        "    return new Order(customer, new BigDecimal(total), status);",
+        "  }",
+        "",
+        "  private final List<Order> orders =",
+        "      List.of(",
+        '          order("zoe", "10.00", Order.Status.PAID),',
+        '          order("mia", "2.50", Order.Status.OPEN),',
+        '          order("zoe", "5.25", Order.Status.CANCELLED),',
+        '          order("adam", "7.10", Order.Status.PAID),',
+        '          order("mia", "0.40", Order.Status.PAID),',
+        '          order("bob", "3.00", Order.Status.CANCELLED));',
+        "",
+        "  @Test",
+        "  void skipsCancelledOrders() {",
+        "    Map<String, BigDecimal> totals = OrderReport.totalsByCustomer(orders);",
+        '    assertEquals(0, new BigDecimal("10.00").compareTo(totals.get("zoe")));',
+        '    assertEquals(0, new BigDecimal("2.90").compareTo(totals.get("mia")));',
+        '    assertFalse(totals.containsKey("bob"));',
+        "  }",
+        "",
+        "  @Test",
+        "  void listsCustomersInNameOrder() {",
+        "    Map<String, BigDecimal> totals = OrderReport.totalsByCustomer(orders);",
+        '    assertEquals(List.of("adam", "mia", "zoe"), new ArrayList<>(totals.keySet()));',
+        "  }",
+        "",
+        "  @Test",
+        "  void anEmptyListGivesAnEmptyReport() {",
+        "    assertTrue(OrderReport.totalsByCustomer(List.of()).isEmpty());",
+        "  }",
+        "}"
+      )
+    },
+    solution: {
+      [main("shop/OrderReport.java")]: report(
+        [
+          "    return orders.stream()",
+          "        .filter(order -> order.status() != Order.Status.CANCELLED)",
+          "        .collect(",
+          "            Collectors.groupingBy(",
+          "                Order::customer,",
+          "                TreeMap::new,",
+          "                Collectors.reducing(BigDecimal.ZERO, Order::total, BigDecimal::add)));"
+        ],
+        [
+          "import java.math.BigDecimal;",
+          "import java.util.List;",
+          "import java.util.Map;",
+          "import java.util.TreeMap;",
+          "import java.util.stream.Collectors;"
+        ]
+      )
+    }
+  })
+];
+
+// src/evals/pythonTasks.ts
+var task3 = (t) => ({
+  ...t,
+  files: { "pyproject.toml": PYPROJECT, ...t.files },
+  check: PYTEST_CHECK,
+  requires: ["pytest"]
+});
+var cart = (init, body) => lines(
+  "class Cart:",
+  '    """A shopping cart. Each cart has its own items."""',
+  "",
+  `    def __init__(self, items=${init}):`,
+  `        ${body}`,
+  "",
+  "    def add(self, item):",
+  "        self.items.append(item)",
+  "",
+  "    def count(self):",
+  "        return len(self.items)"
+);
+var chunks = (stop) => lines(
+  "def chunks(seq, size):",
+  '    """Split seq into lists of `size` items. The last list may be shorter."""',
+  "    if size < 1:",
+  '        raise ValueError("size must be positive")',
+  `    return [list(seq[i : i + size]) for i in range(0, ${stop}, size)]`
+);
+var users = (name) => lines(
+  'NAMES = {1: "ada", 2: "guido"}',
+  "",
+  "",
+  `def ${name}(user_id):`,
+  '    return NAMES.get(user_id, "unknown")'
+);
+var profile2 = (name) => lines(
+  `from app.users import ${name}`,
+  "",
+  "",
+  "def title(user_id):",
+  `    return f"Profile of {${name}(user_id)}"`
+);
+var admin2 = (name) => lines(
+  "from app import users",
+  "",
+  "",
+  "def is_admin(user_id):",
+  `    return users.${name}(user_id) == "ada"`
+);
+var top = (key, reverse) => lines(
+  "def top_players(scores, n):",
+  '    """The n best players: highest score first; equal scores in name order."""',
+  `    return sorted(scores, key=${key}${reverse})[:n]`
+);
+var PYTHON_TASKS = [
+  task3({
+    id: "py-mutable-default",
+    title: "Python: carts share their items",
+    prompt: "Two Cart objects share the same items, and a test fails. Fix Cart. Do not change the tests.",
+    files: {
+      "src/shop/__init__.py": "",
+      "src/shop/cart.py": cart("[]", "self.items = items"),
+      "tests/test_cart.py": lines(
+        "from shop.cart import Cart",
+        "",
+        "",
+        "def test_new_carts_are_empty_and_separate():",
+        "    first = Cart()",
+        '    first.add("tea")',
+        "    second = Cart()",
+        "    assert second.count() == 0",
+        "    assert first.count() == 1",
+        "",
+        "",
+        "def test_a_cart_can_start_with_items():",
+        '    items = ["milk"]',
+        "    cart = Cart(items)",
+        '    cart.add("bread")',
+        "    assert cart.count() == 2"
+      )
+    },
+    solution: {
+      "src/shop/cart.py": cart("None", "self.items = [] if items is None else items")
+    }
+  }),
+  task3({
+    id: "py-chunks",
+    title: "Python: keep the last short chunk",
+    prompt: "chunks() loses the last items when they do not fill a whole chunk. Fix it without changing the tests.",
+    files: {
+      "src/listutil/__init__.py": "from listutil.chunks import chunks\n",
+      "src/listutil/chunks.py": chunks("len(seq) - size + 1"),
+      "tests/test_chunks.py": lines(
+        "import pytest",
+        "",
+        "from listutil import chunks",
+        "",
+        "",
+        "def test_even_split():",
+        "    assert chunks([1, 2, 3, 4], 2) == [[1, 2], [3, 4]]",
+        "",
+        "",
+        "def test_last_chunk_may_be_short():",
+        "    assert chunks([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4], [5]]",
+        '    assert chunks("abc", 5) == [["a", "b", "c"]]',
+        "",
+        "",
+        "def test_empty_input():",
+        "    assert chunks([], 3) == []",
+        "",
+        "",
+        "def test_rejects_a_bad_size():",
+        "    with pytest.raises(ValueError):",
+        "        chunks([1], 0)"
+      )
+    },
+    solution: { "src/listutil/chunks.py": chunks("len(seq)") }
+  }),
+  task3({
+    id: "py-rename",
+    title: "Python: rename a function in every module",
+    prompt: "Rename the function get_usr to get_user everywhere in src/ (the definition and every use). Then run the tests.",
+    files: {
+      "src/app/__init__.py": "",
+      "src/app/users.py": users("get_usr"),
+      "src/app/profile.py": profile2("get_usr"),
+      "src/app/admin.py": admin2("get_usr"),
+      "tests/test_app.py": lines(
+        "from app.admin import is_admin",
+        "from app.profile import title",
+        "from app.users import get_user",
+        "",
+        "",
+        "def test_users():",
+        '    assert get_user(1) == "ada"',
+        '    assert get_user(9) == "unknown"',
+        "",
+        "",
+        "def test_services():",
+        '    assert title(2) == "Profile of guido"',
+        "    assert is_admin(1)"
+      )
+    },
+    solution: {
+      "src/app/users.py": users("get_user"),
+      "src/app/profile.py": profile2("get_user"),
+      "src/app/admin.py": admin2("get_user")
+    }
+  }),
+  task3({
+    id: "py-duration",
+    title: "Python: implement a duration parser from its tests",
+    prompt: "Implement parse_duration in src/timeparse/duration.py so that the tests pass. Read the tests first. Do not change the tests.",
+    files: {
+      "src/timeparse/__init__.py": "",
+      "src/timeparse/duration.py": lines(
+        "def parse_duration(text):",
+        '    """Seconds in a duration such as "1h30m", "45s" or "2h5s"."""',
+        '    raise NotImplementedError("parse_duration")'
+      ),
+      "tests/test_duration.py": lines(
+        "import pytest",
+        "",
+        "from timeparse.duration import parse_duration",
+        "",
+        "",
+        '@pytest.mark.parametrize("text, seconds", [',
+        '    ("45s", 45),',
+        '    ("2m", 120),',
+        '    ("1h30m", 5400),',
+        '    ("2h5s", 7205),',
+        '    ("1h1m1s", 3661),',
+        '    ("0s", 0),',
+        "])",
+        "def test_valid(text, seconds):",
+        "    assert parse_duration(text) == seconds",
+        "",
+        "",
+        '@pytest.mark.parametrize("text", ["", "10", "5x", "1m1h", "1h1h", "h", "1 h", "-1s"])',
+        "def test_invalid(text):",
+        "    with pytest.raises(ValueError):",
+        "        parse_duration(text)"
+      )
+    },
+    solution: {
+      "src/timeparse/duration.py": lines(
+        "import re",
+        "",
+        'PATTERN = re.compile(r"(?:(\\d+)h)?(?:(\\d+)m)?(?:(\\d+)s)?")',
+        "",
+        "",
+        "def parse_duration(text):",
+        "    match = PATTERN.fullmatch(text)",
+        "    if not text or match is None:",
+        '        raise ValueError(f"not a duration: {text!r}")',
+        "    hours, minutes, seconds = (int(g) if g else 0 for g in match.groups())",
+        "    return hours * 3600 + minutes * 60 + seconds"
+      )
+    }
+  }),
+  task3({
+    id: "py-top-players",
+    title: "Python: fix the order of equal scores",
+    prompt: "top_players returns players with equal scores in the wrong order, and a test fails. Fix it without changing the tests.",
+    files: {
+      "src/league/__init__.py": "",
+      "src/league/ranking.py": top("scores.get", ", reverse=True"),
+      "tests/test_ranking.py": lines(
+        "from league.ranking import top_players",
+        "",
+        'SCORES = {"zed": 5, "amy": 5, "bob": 7, "cat": 1}',
+        "",
+        "",
+        "def test_best_first_and_ties_in_name_order():",
+        '    assert top_players(SCORES, 3) == ["bob", "amy", "zed"]',
+        "",
+        "",
+        "def test_n_larger_than_the_list():",
+        '    assert top_players(SCORES, 10) == ["bob", "amy", "zed", "cat"]',
+        "",
+        "",
+        "def test_zero():",
+        "    assert top_players(SCORES, 0) == []"
+      )
+    },
+    solution: {
+      "src/league/ranking.py": top("lambda name: (-scores[name], name)", "")
+    }
+  })
+];
+
+// src/evals/tasks.ts
+var pkg = '{ "name": "eval", "type": "module", "private": true }\n';
+var lines3 = (...l) => `${l.join("\n")}
+`;
+var imports = (from, names) => lines3(
+  'import assert from "node:assert/strict";',
+  'import { test } from "node:test";',
+  `import { ${names} } from "${from}";`
+);
+var EVAL_TASKS = [
+  {
+    id: "fix-add",
+    title: "Fix a wrong operator",
+    prompt: "The tests fail. Find the bug and fix it. Do not change the tests.",
+    files: {
+      "package.json": pkg,
+      "src/math.js": lines3("export function add(a, b) {", "  return a - b;", "}"),
+      "test/math.test.js": `${imports("../src/math.js", "add")}test("add", () => assert.equal(add(2, 3), 5));
+`
+    },
+    check: "node --test",
+    solution: { "src/math.js": lines3("export function add(a, b) {", "  return a + b;", "}") }
+  },
+  {
+    id: "off-by-one",
+    title: "Fix an off-by-one error",
+    prompt: "range(n) should return [0, 1, \u2026, n-1], but a test fails. Fix it without changing the tests.",
+    files: {
+      "package.json": pkg,
+      "src/range.js": lines3(
+        "export function range(n) {",
+        "  const out = [];",
+        "  for (let i = 0; i < n - 1; i++) out.push(i);",
+        "  return out;",
+        "}"
+      ),
+      "test/range.test.js": `${imports("../src/range.js", "range")}${lines3(
+        'test("range", () => {',
+        "  assert.deepEqual(range(3), [0, 1, 2]);",
+        "  assert.deepEqual(range(0), []);",
+        "});"
+      )}`
+    },
+    check: "node --test",
+    solution: {
+      "src/range.js": lines3(
+        "export function range(n) {",
+        "  const out = [];",
+        "  for (let i = 0; i < n; i++) out.push(i);",
+        "  return out;",
+        "}"
+      )
+    }
+  },
+  {
+    id: "rename",
+    title: "Rename a function in every file",
+    prompt: "Rename the function getUsr to getUser everywhere in src/ (definition and every use). Then run the tests.",
+    files: {
+      "package.json": pkg,
+      "src/users.js": lines3(
+        "export function getUsr(id) {",
+        "  return { id, name: `user-${id}` };",
+        "}"
+      ),
+      "src/profile.js": lines3(
+        'import { getUsr } from "./users.js";',
+        "export const profile = (id) => `Profile of ${getUsr(id).name}`;"
+      ),
+      "src/admin.js": lines3(
+        'import { getUsr } from "./users.js";',
+        "export const isAdmin = (id) => getUsr(id).id === 1;"
+      ),
+      "test/users.test.js": `${imports("../src/users.js", "getUser")}${lines3(
+        'import { profile } from "../src/profile.js";',
+        'import { isAdmin } from "../src/admin.js";',
+        'test("rename", () => {',
+        '  assert.equal(getUser(2).name, "user-2");',
+        '  assert.equal(profile(3), "Profile of user-3");',
+        "  assert.equal(isAdmin(1), true);",
+        "});"
+      )}`
+    },
+    check: "node --test && ! grep -rq getUsr src",
+    solution: {
+      "src/users.js": lines3(
+        "export function getUser(id) {",
+        "  return { id, name: `user-${id}` };",
+        "}"
+      ),
+      "src/profile.js": lines3(
+        'import { getUser } from "./users.js";',
+        "export const profile = (id) => `Profile of ${getUser(id).name}`;"
+      ),
+      "src/admin.js": lines3(
+        'import { getUser } from "./users.js";',
+        "export const isAdmin = (id) => getUser(id).id === 1;"
+      )
+    }
+  },
+  {
+    id: "implement-slugify",
+    title: "Implement a function from its tests",
+    prompt: "Implement slugify in src/slugify.js so that the tests pass.",
+    files: {
+      "package.json": pkg,
+      "src/slugify.js": lines3(
+        "export function slugify(text) {",
+        '  throw new Error("not implemented");',
+        "}"
+      ),
+      "test/slugify.test.js": `${imports("../src/slugify.js", "slugify")}${lines3(
+        'test("slugify", () => {',
+        '  assert.equal(slugify("Hello World"), "hello-world");',
+        '  assert.equal(slugify("  Many   spaces  "), "many-spaces");',
+        '  assert.equal(slugify("Caf\xE9 & Cr\xE8me!"), "cafe-creme");',
+        '  assert.equal(slugify("a--b"), "a-b");',
+        "});"
+      )}`
+    },
+    check: "node --test",
+    solution: {
+      "src/slugify.js": lines3(
+        "export function slugify(text) {",
+        "  return text",
+        '    .normalize("NFKD")',
+        '    .replace(/[\\u0300-\\u036f]/g, "")',
+        "    .toLowerCase()",
+        '    .replace(/[^a-z0-9]+/g, "-")',
+        '    .replace(/^-+|-+$/g, "");',
+        "}"
+      )
+    }
+  },
+  {
+    id: "missing-await",
+    title: "Fix a missing await",
+    prompt: "loadTotal sometimes returns the wrong value and its test fails. Find the cause and fix it.",
+    files: {
+      "package.json": pkg,
+      "src/total.js": lines3(
+        "const fetchPrices = async () => {",
+        "  await new Promise((r) => setTimeout(r, 5));",
+        "  return [3, 4, 5];",
+        "};",
+        "",
+        "export async function loadTotal() {",
+        "  const prices = fetchPrices();",
+        "  let total = 0;",
+        "  for (const p of prices ?? []) total += p;",
+        "  return total;",
+        "}"
+      ),
+      "test/total.test.js": `${imports("../src/total.js", "loadTotal")}test("total", async () => assert.equal(await loadTotal(), 12));
+`
+    },
+    check: "node --test",
+    solution: {
+      "src/total.js": lines3(
+        "const fetchPrices = async () => {",
+        "  await new Promise((r) => setTimeout(r, 5));",
+        "  return [3, 4, 5];",
+        "};",
+        "",
+        "export async function loadTotal() {",
+        "  const prices = await fetchPrices();",
+        "  let total = 0;",
+        "  for (const p of prices ?? []) total += p;",
+        "  return total;",
+        "}"
+      )
+    }
+  },
+  {
+    id: "csv-quotes",
+    title: "Handle quoted fields in a CSV parser",
+    prompt: 'parseLine splits a CSV line at commas. It must also handle fields in double quotes that contain commas and escaped quotes (""). Fix it so the tests pass.',
+    files: {
+      "package.json": pkg,
+      "src/csv.js": lines3("export function parseLine(line) {", '  return line.split(",");', "}"),
+      "test/csv.test.js": `${imports("../src/csv.js", "parseLine")}${lines3(
+        'test("plain", () => assert.deepEqual(parseLine("a,b,c"), ["a", "b", "c"]));',
+        `test("quoted comma", () => assert.deepEqual(parseLine('x,"a, b",y'), ["x", "a, b", "y"]));`,
+        `test("escaped quote", () => assert.deepEqual(parseLine('"say ""hi""",z'), ['say "hi"', "z"]));`,
+        'test("empty fields", () => assert.deepEqual(parseLine("a,,b"), ["a", "", "b"]));'
+      )}`
+    },
+    check: "node --test",
+    solution: {
+      "src/csv.js": lines3(
+        "export function parseLine(line) {",
+        "  const out = [];",
+        '  let field = "";',
+        "  let quoted = false;",
+        "  for (let i = 0; i < line.length; i++) {",
+        "    const c = line[i];",
+        "    if (quoted) {",
+        `      if (c === '"' && line[i + 1] === '"') { field += '"'; i++; }`,
+        `      else if (c === '"') quoted = false;`,
+        "      else field += c;",
+        `    } else if (c === '"') quoted = true;`,
+        '    else if (c === ",") { out.push(field); field = ""; }',
+        "    else field += c;",
+        "  }",
+        "  out.push(field);",
+        "  return out;",
+        "}"
+      )
+    }
+  },
+  {
+    id: "write-tests",
+    title: "Write tests for existing code",
+    prompt: "Write a test file test/isPrime.test.js for src/isPrime.js with node:test. Cover at least 0, 1, 2, a prime above 10 and a composite number. Run it.",
+    files: {
+      "package.json": pkg,
+      "src/isPrime.js": lines3(
+        "export function isPrime(n) {",
+        "  if (n < 2) return false;",
+        "  for (let d = 2; d * d <= n; d++) if (n % d === 0) return false;",
+        "  return true;",
+        "}"
+      )
+    },
+    check: 'test -f test/isPrime.test.js && node --test && [ "$(grep -c "assert" test/isPrime.test.js)" -ge 4 ]',
+    protect: ["src/isPrime.js"],
+    solution: {
+      "test/isPrime.test.js": `${imports("../src/isPrime.js", "isPrime")}${lines3(
+        'test("isPrime", () => {',
+        "  assert.equal(isPrime(0), false);",
+        "  assert.equal(isPrime(1), false);",
+        "  assert.equal(isPrime(2), true);",
+        "  assert.equal(isPrime(13), true);",
+        "  assert.equal(isPrime(15), false);",
+        "});"
+      )}`
+    }
+  },
+  {
+    id: "config-default",
+    title: "Add a default value",
+    prompt: "loadConfig must use port 3000 when the config has no port, but keep a port that is set. Make the tests pass.",
+    files: {
+      "package.json": pkg,
+      "src/config.js": lines3(
+        "export function loadConfig(json) {",
+        "  const config = JSON.parse(json);",
+        "  return config;",
+        "}"
+      ),
+      "test/config.test.js": `${imports("../src/config.js", "loadConfig")}${lines3(
+        `test("default port", () => assert.equal(loadConfig('{"host":"x"}').port, 3000));`,
+        `test("set port", () => assert.equal(loadConfig('{"port":8080}').port, 8080));`,
+        `test("keeps other keys", () => assert.equal(loadConfig('{"host":"x"}').host, "x"));`
+      )}`
+    },
+    check: "node --test",
+    solution: {
+      "src/config.js": lines3(
+        "export function loadConfig(json) {",
+        "  const config = JSON.parse(json);",
+        "  return { port: 3000, ...config };",
+        "}"
+      )
+    }
+  },
+  {
+    id: "new-module",
+    title: "Create a new module",
+    prompt: 'Create src/version.js that exports a constant VERSION with the "version" value from package.json (read the file; do not import JSON). The test must pass.',
+    files: {
+      "package.json": '{ "name": "eval", "version": "1.2.3", "type": "module", "private": true }\n',
+      "test/version.test.js": `${imports("../src/version.js", "VERSION")}test("version", () => assert.equal(VERSION, "1.2.3"));
+`
+    },
+    check: "node --test",
+    solution: {
+      "src/version.js": lines3(
+        'import { readFileSync } from "node:fs";',
+        'const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));',
+        "export const VERSION = pkg.version;"
+      )
+    }
+  },
+  {
+    id: "extract-helper",
+    title: "Remove duplicate code",
+    prompt: "signup.js and invite.js have the same e-mail check. Move it into a function isValidEmail(text) exported from src/email.js, and use it in both files. The tests must still pass.",
+    files: {
+      "package.json": pkg,
+      "src/signup.js": lines3(
+        "export function signup(email) {",
+        '  if (!/^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$/.test(email)) throw new Error("bad email");',
+        "  return { email };",
+        "}"
+      ),
+      "src/invite.js": lines3(
+        "export function invite(email) {",
+        '  if (!/^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$/.test(email)) throw new Error("bad email");',
+        "  return { invited: email };",
+        "}"
+      ),
+      "test/email.test.js": `${imports("../src/signup.js", "signup")}${lines3(
+        'import { invite } from "../src/invite.js";',
+        'test("signup", () => { assert.equal(signup("a@b.co").email, "a@b.co"); assert.throws(() => signup("nope")); });',
+        'test("invite", () => { assert.equal(invite("a@b.co").invited, "a@b.co"); assert.throws(() => invite("x@y")); });'
+      )}`
+    },
+    check: [
+      "node --test",
+      `node -e 'import("./src/email.js").then((m) => process.exit(m.isValidEmail("a@b.co") && !m.isValidEmail("x") ? 0 : 1))'`,
+      "grep -q isValidEmail src/signup.js",
+      "grep -q isValidEmail src/invite.js",
+      // The e-mail pattern must appear only once in src/ after the change.
+      '[ "$(cat src/*.js | grep -c "\\[^@")" -le 1 ]'
+    ].join(" && "),
+    solution: {
+      "src/email.js": lines3(
+        "export function isValidEmail(text) {",
+        "  return /^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$/.test(text);",
+        "}"
+      ),
+      "src/signup.js": lines3(
+        'import { isValidEmail } from "./email.js";',
+        "export function signup(email) {",
+        '  if (!isValidEmail(email)) throw new Error("bad email");',
+        "  return { email };",
+        "}"
+      ),
+      "src/invite.js": lines3(
+        'import { isValidEmail } from "./email.js";',
+        "export function invite(email) {",
+        '  if (!isValidEmail(email)) throw new Error("bad email");',
+        "  return { invited: email };",
+        "}"
+      )
+    }
+  }
+];
+
+// src/evals/suites.ts
+var EVAL_SUITES = {
+  basic: EVAL_TASKS,
+  hard: HARD_TASKS,
+  java: JAVA_TASKS,
+  python: PYTHON_TASKS
+};
+var ALL_TASKS = Object.values(EVAL_SUITES).flat();
+function requiredToolchains(tasks) {
+  return [...new Set(tasks.flatMap((t) => t.requires ?? []))];
+}
+
+// src/cli/evalCommand.ts
+var BATCH_TASK_TIMEOUT_MS = 12 * 60 * 6e4;
+async function runEvalCommand(options) {
+  if (options.fromGit) return buildFromGit(options);
+  let repoTasks;
+  if (options.list || options.suite === "repo") {
+    const loaded = await repoSuite();
+    if (typeof loaded === "string") {
+      if (options.suite === "repo") {
+        process.stderr.write(`${loaded}
+`);
+        return 1;
+      }
+    } else repoTasks = repoEvalTasks(loaded);
+  }
+  if (options.list) {
+    const suites = [
+      ...Object.entries(EVAL_SUITES),
+      ...repoTasks === void 0 ? [] : [["repo", repoTasks]]
+    ];
+    for (const [suite2, tasks2] of suites) {
+      const needs = requiredToolchains(tasks2).map((id) => TOOLCHAINS[id].title);
+      process.stdout.write(`${suite2}:${needs.length > 0 ? ` (needs ${needs.join(", ")})` : ""}
+`);
+      for (const task4 of tasks2) process.stdout.write(`  ${task4.id.padEnd(18)} ${task4.title}
+`);
+    }
+    return 0;
+  }
+  if (options.prepare !== void 0) return prepare(options.prepare);
+  const modelId = options.model;
+  if (!modelId) {
+    process.stderr.write("Set a model with --model <id> or the GARUDA_MODEL variable.\n");
+    return 1;
+  }
+  const models = await loadModelsConfig();
+  if (models.problem !== void 0) {
+    process.stderr.write(`${models.problem}
+`);
+    return 1;
+  }
+  let resolved;
+  try {
+    resolved = resolveModel(modelId, models.config);
+  } catch (error) {
+    process.stderr.write(`${error.message}
+`);
+    return 1;
+  }
+  for (const note of resolved.notes) process.stderr.write(`${note}
+`);
+  const index = options.index ?? DEFAULT_CODE_INDEX_MODE;
+  if (!CODE_INDEX_MODES.includes(index)) {
+    process.stderr.write(
+      `Unknown index mode ${options.index}. Use: ${CODE_INDEX_MODES.join(", ")}.
+`
+    );
+    return 1;
+  }
+  if (options.executor !== void 0 && !EXECUTOR_NAMES.includes(options.executor)) {
+    process.stderr.write(
+      `Unknown executor ${options.executor}. Use: ${EXECUTOR_NAMES.join(", ")}.
+`
+    );
+    return 1;
+  }
+  const subagentsMode = options.subagents ?? "off";
+  if (subagentsMode !== "on" && subagentsMode !== "off") {
+    process.stderr.write(`Unknown subagents mode ${options.subagents}. Use: on, off.
+`);
+    return 1;
+  }
+  const todoMode = options.todo ?? "off";
+  if (todoMode !== "on" && todoMode !== "off") {
+    process.stderr.write(`Unknown todo mode ${options.todo}. Use: on, off.
+`);
+    return 1;
+  }
+  const formatMode = options.format ?? "off";
+  if (formatMode !== "on" && formatMode !== "off") {
+    process.stderr.write(`Unknown format mode ${options.format}. Use: on, off.
+`);
+    return 1;
+  }
+  let biome;
+  if (options.format !== void 0) {
+    biome = garudaBiome();
+    if (biome === void 0) {
+      process.stderr.write(
+        "--format needs Biome in Garuda's node_modules (a source checkout after pnpm install).\n"
+      );
+      return 1;
+    }
+    if (options.suite === "java" || options.suite === "python" || options.suite === "all" || options.suite === "repo") {
+      process.stderr.write("--format works with the JS suites: basic and hard.\n");
+      return 1;
+    }
+  }
+  const thinkingMode = options.keepThinking ?? "on";
+  if (thinkingMode !== "on" && thinkingMode !== "off") {
+    process.stderr.write(`Unknown keep-thinking mode ${options.keepThinking}. Use: on, off.
+`);
+    return 1;
+  }
+  const lspMode = options.lsp ?? "off";
+  if (lspMode !== "on" && lspMode !== "off") {
+    process.stderr.write(`Unknown lsp mode ${options.lsp}. Use: on, off.
+`);
+    return 1;
+  }
+  let network;
+  if (options.network !== void 0) {
+    network = options.network.flatMap((e) => e.split(",")).filter((e) => e.trim() !== "");
+    const { problems } = expandAllowlist(network);
+    if (problems.length > 0) {
+      process.stderr.write(`--network: ${problems.join(" ")}
+`);
+      return 1;
+    }
+    if (network.length === 0) network = void 0;
+  }
+  const batchMode = options.batch ?? "off";
+  if (batchMode !== "on" && batchMode !== "off") {
+    process.stderr.write(`Unknown batch mode ${options.batch}. Use: on, off.
+`);
+    return 1;
+  }
+  const createBatch = resolved.createBatch;
+  if (batchMode === "on" && createBatch === void 0) {
+    process.stderr.write(`--batch on needs an Anthropic model; ${resolved.spec} is not one.
+`);
+    return 1;
+  }
+  let sub;
+  if (options.subagentModel) {
+    try {
+      sub = resolveModel(options.subagentModel, models.config);
+    } catch (error) {
+      process.stderr.write(`${error.message}
+`);
+      return 1;
+    }
+  }
+  const { assertModelAllowedByPolicy, loadTeamPolicy } = await import("./policy-EJPLEEXP.js");
+  let team;
+  try {
+    team = await loadTeamPolicy();
+    assertModelAllowedByPolicy(team?.policy, resolved.spec);
+    if (sub !== void 0) assertModelAllowedByPolicy(team?.policy, sub.spec);
+  } catch (error) {
+    process.stderr.write(`Team policy: ${error.message}
+`);
+    return 1;
+  }
+  const suite = options.suite ?? "basic";
+  const suiteTasks = suite === "all" ? ALL_TASKS : suite === "repo" ? repoTasks : EVAL_SUITES[suite];
+  if (suiteTasks === void 0) {
+    process.stderr.write(
+      `Unknown suite ${suite}. Use: ${[...Object.keys(EVAL_SUITES), "all", "repo"].join(", ")}.
+`
+    );
+    return 1;
+  }
+  const pool = suite === "repo" ? suiteTasks : ALL_TASKS;
+  const chosen = options.task === void 0 ? suiteTasks : pool.filter((t) => options.task?.includes(t.id));
+  const repeat = Math.max(1, options.repeat ?? 1);
+  const tasks = chosen.flatMap((t) => Array.from({ length: repeat }, () => t));
+  if (chosen.length === 0) {
+    process.stderr.write("No task matches. Use --list to see the task ids.\n");
+    return 1;
+  }
+  const missing = (await checkToolchains(requiredToolchains(chosen))).filter((s) => !s.ok);
+  if (missing.length > 0) {
+    for (const status of missing) {
+      const detail = status.output.split("\n").filter((line) => line.trim() !== "").slice(0, 2).join("\n    ");
+      process.stderr.write(
+        `${TOOLCHAINS[status.id].hint}
+${detail === "" ? "" : `    ${detail}
+`}`
+      );
+    }
+    return 1;
+  }
+  const { executor, notice } = createExecutor(options.executor ?? "auto");
+  if (notice !== void 0) process.stderr.write(`${notice}
+`);
+  if (lspMode === "on") {
+    if (executor.isolation === "none") {
+      process.stderr.write("--lsp on needs the OS sandbox: language servers run only there.\n");
+      return 1;
+    }
+    const { discoverServer, LSP_LANGUAGES } = await import("./servers-6YDGWOZ6.js");
+    const found = LSP_LANGUAGES.map((l) => ({ l, s: discoverServer(l, { root: process.cwd() }) }));
+    if (found.every((f) => f.s === void 0)) {
+      process.stderr.write(
+        "--lsp on: no language server found. Run: garuda lsp install typescript\n"
+      );
+      return 1;
+    }
+    for (const { l, s } of found) {
+      process.stderr.write(
+        `LSP ${l}: ${s === void 0 ? "none" : `${s.spec.name} (${s.path})`}
+`
+      );
+    }
+  }
+  const where = executor.isolation === "none" ? "commands run on this machine with no sandbox" : `commands and the checks run in the ${executor.name} sandbox`;
+  const parallel = Math.max(
+    1,
+    options.parallel ?? (batchMode === "on" ? Math.min(tasks.length, 20) : 1)
+  );
+  const outDir = join5(process.cwd(), ".garuda", "evals", newSessionId());
+  mkdirSync3(outDir, { recursive: true });
+  process.stderr.write(
+    `Running ${chosen.length} task(s)${repeat > 1 ? ` \xD7 ${repeat}` : ""} (suite ${options.task === void 0 ? suite : "custom"}, code index ${index}, subagents ${subagentsMode}${sub === void 0 ? "" : ` on ${sub.spec}`}, todo ${todoMode}, lsp ${lspMode}, keep-thinking ${thinkingMode}, format ${formatMode}, network ${network === void 0 ? "off" : network.join(",")}, batch ${batchMode}${parallel > 1 ? `, ${parallel} at a time` : ""}) with ${modelId}. Garuda approves every call except its deny rules;
+${where}, in scratch folders.
+
+`
+  );
+  if (batchMode === "on") {
+    process.stderr.write(
+      "Batch API: each model call waits for its batch (minutes, sometimes more) at half the token price.\n\n"
+    );
+  }
+  const started = Date.now();
+  const results = await runEvals(
+    tasks,
+    {
+      modelId: resolved.spec,
+      ...team === void 0 ? {} : { policy: team.policy },
+      model: () => batchMode === "on" && createBatch !== void 0 ? createBatch() : resolved.create(),
+      // Batch responses carry their own price factor (half): the price stays the model's.
+      modelInfo: resolved.info,
+      parallel,
+      ...batchMode === "on" ? { taskTimeoutMs: BATCH_TASK_TIMEOUT_MS } : {},
+      ...resolved.maxTokens === void 0 ? {} : { maxTokens: resolved.maxTokens },
+      outDir,
+      ...options.maxSteps === void 0 ? {} : { maxSteps: options.maxSteps },
+      codeIndex: index,
+      subagents: subagentsMode === "on",
+      todo: todoMode === "on",
+      lsp: lspMode === "on",
+      keepThinking: thinkingMode === "on",
+      ...network === void 0 ? {} : { network },
+      ...biome === void 0 ? {} : { format: { on: formatMode === "on", biome } },
+      ...sub === void 0 ? {} : { subagentModel: { spec: sub.spec, model: () => sub.create(), info: sub.info } },
+      executor: executor.name === "host" ? "host" : "os",
+      ...options.keep ? { keep: true } : {},
+      onEvent: (taskId, event) => {
+        if (event.type === "tool_call") process.stderr.write(`  [${taskId}] ${event.call.name}
+`);
+        if (event.type === "notice") process.stderr.write(`  [${taskId}] ${event.text}
+`);
+        if (event.type === "model_retry")
+          process.stderr.write(
+            `  [${taskId}] retry ${event.attempt}/${event.maxRetries}: ${event.reason}
+`
+          );
+      }
+    },
+    (r) => process.stderr.write(
+      `${r.passed ? "\u2713" : "\u2717"} ${r.id}: ${r.passed ? "pass" : "fail"} (${r.steps} steps)${r.reason ? `
+    ${r.reason.split("\n").slice(0, 3).join("\n    ")}` : ""}
+`
+    )
+  );
+  const wallMs = Date.now() - started;
+  const report2 = `${formatReport(results)}
+Wall time: ${Math.round(wallMs / 1e3)} s${parallel > 1 ? ` (${parallel} tasks at a time)` : ""}.`;
+  writeFileSync(
+    join5(outDir, "report.json"),
+    `${JSON.stringify({ model: modelId, suite, repeat, codeIndex: index, subagents: subagentsMode, subagentModel: sub?.spec, todo: todoMode, lsp: lspMode, keepThinking: thinkingMode, format: formatMode, network: network ?? "off", batch: batchMode, parallel, wallMs, executor: executor.name, results }, null, 2)}
+`
+  );
+  process.stdout.write(`
+${report2}
+
+Session files and report.json: ${outDir}
+`);
+  return results.every((r) => r.passed) ? 0 : 2;
+}
+async function repoSuite() {
+  const root = await gitRoot(process.cwd());
+  if (root === void 0) return "-s repo needs a git repository: run it in your project.";
+  let suite;
+  try {
+    suite = await loadRepoSuite(root);
+  } catch (error) {
+    return error.message;
+  }
+  if (suite === void 0) {
+    return `No repo suite in ${root}. Build it first: garuda eval --from-git`;
+  }
+  if (suite.tasks.length === 0) {
+    return `The repo suite has no tasks. Build it again with more commits: garuda eval --from-git --commits 500`;
+  }
+  return { ...suite, root };
+}
+async function gitRoot(cwd) {
+  const r = await git(new HostExecutor(), cwd, ["rev-parse", "--show-toplevel"], { check: false });
+  return r.exitCode === 0 ? realpathSync3(r.stdout.trim()) : void 0;
+}
+async function buildFromGit(options) {
+  const root = await gitRoot(process.cwd());
+  if (root === void 0) {
+    process.stderr.write("--from-git needs a git repository: run it in your project.\n");
+    return 1;
+  }
+  const testCommand = options.testCommand ?? detectTestCommand(root, detectProfiles(root));
+  if (testCommand === void 0) {
+    process.stderr.write(
+      'No test command found. Give one: garuda eval --from-git --test-command "npm test"\n'
+    );
+    return 1;
+  }
+  for (const [name, value] of [
+    ["--commits", options.commits],
+    ["--max-tasks", options.maxTasks]
+  ]) {
+    if (value !== void 0 && !(Number.isInteger(value) && value > 0)) {
+      process.stderr.write(`${name} must be a whole number above 0.
+`);
+      return 1;
+    }
+  }
+  process.stderr.write(
+    `Building the repo suite from ${options.commits ?? DEFAULT_COMMITS} recent commits${options.since === void 0 ? "" : ` since ${options.since}`} in ${root}.
+Each candidate runs its own test files with \`${testCommand}\` twice on this machine (no sandbox), in a git worktree; your checkout does not change.
+
+`
+  );
+  const { suite, checked, skipped, failure } = await buildRepoSuite(new HostExecutor(), root, {
+    testCommand,
+    ...options.commits === void 0 ? {} : { commits: options.commits },
+    ...options.since === void 0 ? {} : { since: options.since },
+    ...options.maxTasks === void 0 ? {} : { maxTasks: options.maxTasks },
+    onProgress: (line) => process.stderr.write(`  ${line}
+`)
+  });
+  const file = await saveRepoSuite(suite);
+  const reasons = Object.entries(skipped).sort((a, b) => b[1] - a[1]).map(([why, n]) => `  ${String(n).padStart(4)}  ${why}`);
+  process.stdout.write(
+    `${[
+      `
+${suite.tasks.length} task(s) kept; ${checked} candidate(s) checked with the tests.`,
+      ...reasons.length === 0 ? [] : ["Skipped:", ...reasons],
+      ...failure === void 0 ? [] : [
+        `The first failure at a commit (${failure.sha.slice(0, 7)}: \`${failure.command}\`):`,
+        ...failure.tail.split("\n").map((line) => `  ${line}`),
+        'If every commit fails like this, the tests do not run in a worktree on this machine; give a working command with --test-command ("{files}" takes the test files).'
+      ],
+      `Suite: ${file}`,
+      ...suite.tasks.length === 0 ? ["No task: try more commits (--commits 500) or an older --since."] : ["Run it: garuda eval -s repo -m <model> (add --list to see the tasks)"]
+    ].join("\n")}
+`
+  );
+  return suite.tasks.length === 0 ? 2 : 0;
+}
+async function prepare(name) {
+  const id = toolchainId(name);
+  if (id === void 0) {
+    process.stderr.write(`Unknown toolchain ${name}. Use: java, python.
+`);
+    return 1;
+  }
+  const toolchain = TOOLCHAINS[id];
+  if (toolchain.prepare !== void 0) {
+    process.stderr.write(
+      `Preparing ${toolchain.title}: running \`${toolchain.prepare}\` in a scratch project (downloads with network)\u2026
+`
+    );
+    const r = await prepareToolchain(id);
+    if (!r.ok) {
+      process.stderr.write(`It failed:
+${r.output}
+`);
+      return 1;
+    }
+  }
+  const [status] = await checkToolchains([id]);
+  if (status?.ok) {
+    process.stderr.write(`${toolchain.title}: ready.
+`);
+    return 0;
+  }
+  process.stderr.write(`${toolchain.hint}
+${status?.output ?? ""}
+`);
+  return 1;
+}
+function garudaBiome() {
+  try {
+    const pkg2 = createRequire(import.meta.url).resolve("@biomejs/biome/package.json");
+    const bin = join5(dirname3(pkg2), "bin", "biome");
+    return existsSync2(bin) ? bin : void 0;
+  } catch {
+    return void 0;
+  }
+}
+export {
+  BATCH_TASK_TIMEOUT_MS,
+  runEvalCommand
+};
+//# sourceMappingURL=evalCommand-3SSFMZQD.js.map