@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-3U5TXJZS.js

@@ -1,251 +0,0 @@
-import {
-  preflightVenvImports,
-  readJsonReport,
-  runStagedJudge,
-  runVenvPython,
-  safeRunId,
-  stageFile
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/swe-bench.ts
-import { join } from "path";
-var SWE_REPO_DIR = "/work";
-var swePatchOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)];
-    const last = fences.at(-1)?.[1];
-    return (last ?? text).trim();
-  }
-};
-var DATASET = "princeton-nlp/SWE-bench_Verified";
-var SWE_CACHE_LEVELS = /* @__PURE__ */ new Set(["none", "base", "env", "instance"]);
-function scorerNamespace() {
-  const namespace = process.env.SWEBENCH_NAMESPACE ?? "swebench";
-  if (namespace !== "swebench" && namespace !== "none") {
-    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`);
-  }
-  return namespace;
-}
-var TEST_FILE_EXCLUDES = [
-  "':(exclude,glob)**/tests/**'",
-  "':(exclude,glob)**/test/**'",
-  "':(exclude,glob)test_*.py'",
-  "':(exclude,glob)**/test_*.py'",
-  "':(exclude,glob)*_test.py'",
-  "':(exclude,glob)**/*_test.py'",
-  "':(exclude,glob)conftest.py'",
-  "':(exclude,glob)**/conftest.py'"
-].join(" ");
-function stringIds(report, key) {
-  const value = report[key];
-  if (value === void 0) return [];
-  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
-    throw new Error(`swe-bench: malformed ${key}`);
-  }
-  return value;
-}
-function scoreSweReport(taskId, value) {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    throw new Error("swe-bench: report must be an object");
-  }
-  const report = value;
-  const statusIds = {
-    resolved: stringIds(report, "resolved_ids"),
-    unresolved: stringIds(report, "unresolved_ids"),
-    emptyPatch: stringIds(report, "empty_patch_ids"),
-    completed: stringIds(report, "completed_ids"),
-    incomplete: stringIds(report, "incomplete_ids"),
-    error: stringIds(report, "error_ids")
-  };
-  const submitted = stringIds(report, "submitted_ids");
-  const mentioned = Object.values(statusIds).flat();
-  if (mentioned.some((id) => id !== taskId) || submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId)) {
-    throw new Error(`swe-bench: report identity mismatch for ${taskId}`);
-  }
-  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {
-    throw new Error(`swe-bench: evaluator failed for ${taskId}`);
-  }
-  const outcomes = [
-    statusIds.resolved.includes(taskId),
-    statusIds.unresolved.includes(taskId),
-    statusIds.emptyPatch.includes(taskId)
-  ];
-  if (outcomes.filter(Boolean).length !== 1) {
-    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`);
-  }
-  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {
-    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`);
-  }
-  const resolved = outcomes[0];
-  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) };
-}
-function sweEvaluationArgv(args) {
-  return [
-    "-m",
-    "swebench.harness.run_evaluation",
-    "--dataset_name",
-    DATASET,
-    "--predictions_path",
-    args.predictionsPath,
-    "--run_id",
-    args.runId,
-    "--instance_ids",
-    args.instanceId,
-    "--max_workers",
-    "1",
-    "--namespace",
-    args.namespace ?? scorerNamespace(),
-    "--cache_level",
-    args.cacheLevel
-  ];
-}
-function shellQuote(value) {
-  return `'${value.replace(/'/g, `'\\''`)}'`;
-}
-function sweMetadata(task) {
-  const repo = String(task.metadata?.repo ?? "");
-  const base = String(task.metadata?.base_commit ?? "");
-  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
-    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`);
-  }
-  if (!/^[0-9a-f]{7,40}$/i.test(base)) {
-    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`);
-  }
-  return { repo, base };
-}
-function createSweBenchAdapter(options = {}) {
-  if (options.timeoutMs !== void 0 && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)) throw new Error("swe-bench: timeoutMs must be a positive integer");
-  const cacheLevel = options.cacheLevel ?? "env";
-  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error("swe-bench: invalid cacheLevel");
-  if (options.captureEvaluatorArtifacts !== void 0 && typeof options.captureEvaluatorArtifacts !== "function") throw new Error("swe-bench: captureEvaluatorArtifacts must be a function");
-  let attemptSequence = 0;
-  return {
-    name: "swe-bench-verified",
-    output: swePatchOutput,
-    // Extract the patch from repo STATE, not printed text: stage every edit the
-    // agent made in the cloned repo and diff it against the checked-out
-    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold
-    // `test_patch` itself, so an agent edit to a test would collide on apply. The
-    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold
-    // patch format the swebench judge's `git apply` expects.
-    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the
-    // agent only edits (the harness owns the checkout — a stochastic model can't be
-    // trusted to clone to an exact path). `--quiet` keeps the exec output small.
-    boxSetup(task) {
-      const { repo, base } = sweMetadata(task);
-      return {
-        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`
-      };
-    },
-    boxExtract() {
-      return {
-        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`
-      };
-    },
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["swebench"],
-        requireDocker: true,
-        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; (2) ensure the Docker daemon is running (the judge builds per-instance images).`
-      });
-    },
-    async loadTasks(opts = {}) {
-      const limit = opts.limit ?? 10;
-      const split = opts.split ?? "test";
-      const script = `
-import json, sys
-from datasets import load_dataset
-ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})
-ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
-out = []
-for r in ds:
-    if ids is not None and r["instance_id"] not in ids:
-        continue
-    out.append({
-        "instance_id": r["instance_id"], "repo": r["repo"], "base_commit": r["base_commit"],
-        "problem_statement": r["problem_statement"], "patch": r["patch"], "test_patch": r["test_patch"],
-        "FAIL_TO_PASS": r["FAIL_TO_PASS"], "PASS_TO_PASS": r["PASS_TO_PASS"],
-        "version": r.get("version"), "environment_setup_commit": r.get("environment_setup_commit"),
-    })
-    if ids is None and len(out) >= ${limit}:
-        break
-print(json.dumps(out))
-`;
-      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
-      const rows = JSON.parse(stdout);
-      return rows.map(
-        (r) => ({
-          id: String(r.instance_id),
-          split,
-          prompt: [
-            `Repository: ${r.repo} @ ${r.base_commit}`,
-            "",
-            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
-            "",
-            "Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files \u2014 the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.",
-            "Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff \u2014 the harness reads your committed edits directly from the repo.",
-            "",
-            "--- Issue ---",
-            String(r.problem_statement ?? "")
-          ].join("\n"),
-          metadata: r
-        })
-      );
-    },
-    async goldArtifact(task) {
-      const gold = task.metadata?.patch;
-      return typeof gold === "string" ? gold : void 0;
-    },
-    async judge(task, artifact) {
-      const runId = safeRunId("bench", task.id);
-      const capture = options.captureEvaluatorArtifacts?.({
-        taskId: task.id,
-        runId,
-        attemptSequence: ++attemptSequence
-      });
-      return runStagedJudge({
-        tmpPrefix: "swebench-",
-        ...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
-        ...capture === void 0 ? {} : { capture },
-        // Debug: retain the staged dir (holds swebench's per-instance apply/run
-        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.
-        ...process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {},
-        async stage(dir) {
-          await stageFile(
-            join(dir, "preds.json"),
-            JSON.stringify([
-              { instance_id: task.id, model_name_or_path: "agent-runtime-bench", model_patch: artifact }
-            ])
-          );
-        },
-        // The official evaluation harness. Pulls/builds the instance image, applies
-        // the patch, runs the test spec, writes a per-run report JSON in cwd.
-        argv: (dir) => sweEvaluationArgv({
-          predictionsPath: join(dir, "preds.json"),
-          runId,
-          instanceId: task.id,
-          cacheLevel,
-          namespace: scorerNamespace()
-        }),
-        async parseReport(dir) {
-          const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`));
-          return scoreSweReport(task.id, report);
-        }
-      });
-    }
-  };
-}
-
-export {
-  swePatchOutput,
-  scoreSweReport,
-  sweEvaluationArgv,
-  createSweBenchAdapter
-};
-//# sourceMappingURL=chunk-3U5TXJZS.js.map

package/dist/chunk-3U5TXJZS.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;AAcA,SAAS,YAAY;AAmBrB,IAAM,eAAe;AAWd,IAAM,iBAAwC;AAAA,EACnD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AAGA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,uCAAuC,CAAC;AACzE,UAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC;AAC9B,YAAQ,QAAQ,MAAM,KAAK;AAAA,EAC7B;AACF;AAEA,IAAM,UAAU;AAsBhB,IAAM,mBAAmB,oBAAI,IAAwB,CAAC,QAAQ,QAAQ,OAAO,UAAU,CAAC;AAExF,SAAS,kBAAuC;AAC9C,QAAM,YAAY,QAAQ,IAAI,sBAAsB;AACpD,MAAI,cAAc,cAAc,cAAc,QAAQ;AACpD,UAAM,IAAI,MAAM,kDAAkD,SAAS,GAAG;AAAA,EAChF;AACA,SAAO;AACT;AACA,IAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;AAClF,QAAM,QAAQ,OAAO,GAAG;AACxB,MAAI,UAAU,OAAW,QAAO,CAAC;AACjC,MAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAC,UAAU,OAAO,UAAU,QAAQ,GAAG;AAC7E,UAAM,IAAI,MAAM,wBAAwB,GAAG,EAAE;AAAA,EAC/C;AACA,SAAO;AACT;AAGO,SAAS,eAAe,QAAgB,OAA4B;AACzE,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,UAAM,IAAI,MAAM,qCAAqC;AAAA,EACvD;AACA,QAAM,SAAS;AACf,QAAM,YAAY;AAAA,IAChB,UAAU,UAAU,QAAQ,cAAc;AAAA,IAC1C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,YAAY,UAAU,QAAQ,iBAAiB;AAAA,IAC/C,WAAW,UAAU,QAAQ,eAAe;AAAA,IAC5C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,OAAO,UAAU,QAAQ,WAAW;AAAA,EACtC;AACA,QAAM,YAAY,UAAU,QAAQ,eAAe;AACnD,QAAM,YAAY,OAAO,OAAO,SAAS,EAAE,KAAK;AAChD,MACE,UAAU,KAAK,CAAC,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,CAAC,MAAM,SACxE;AACA,UAAM,IAAI,MAAM,2CAA2C,MAAM,EAAE;AAAA,EACrE;AACA,MAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAAG;AAC7E,UAAM,IAAI,MAAM,mCAAmC,MAAM,EAAE;AAAA,EAC7D;AACA,QAAM,WAAW;AAAA,IACf,UAAU,SAAS,SAAS,MAAM;AAAA,IAClC,UAAU,WAAW,SAAS,MAAM;AAAA,IACpC,UAAU,WAAW,SAAS,MAAM;AAAA,EACtC;AACA,MAAI,SAAS,OAAO,OAAO,EAAE,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,+CAA+C,MAAM,EAAE;AAAA,EACzE;AACA,OAAK,SAAS,CAAC,KAAK,SAAS,CAAC,MAAM,CAAC,UAAU,UAAU,SAAS,MAAM,GAAG;AACzE,UAAM,IAAI,MAAM,sDAAsD,MAAM,EAAE;AAAA,EAChF;AACA,QAAM,WAAW,SAAS,CAAC;AAC3B,SAAO,EAAE,UAAU,OAAO,WAAW,IAAI,GAAG,QAAQ,KAAK,UAAU,MAAM,EAAE;AAC7E;AAEO,SAAS,kBAAkB,MAMrB;AACX,SAAO;AAAA,IACL;AAAA,IAAM;AAAA,IACN;AAAA,IAAkB;AAAA,IAClB;AAAA,IAAsB,KAAK;AAAA,IAC3B;AAAA,IAAY,KAAK;AAAA,IACjB;AAAA,IAAkB,KAAK;AAAA,IACvB;AAAA,IAAiB;AAAA,IACjB;AAAA,IAAe,KAAK,aAAa,gBAAgB;AAAA,IACjD;AAAA,IAAiB,KAAK;AAAA,EACxB;AACF;AAEA,SAAS,WAAW,OAAuB;AACzC,SAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,CAAC;AACzC;AAEA,SAAS,YAAY,MAAiD;AACpE,QAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;AAC7C,QAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;AACpD,MAAI,CAAC,qCAAqC,KAAK,IAAI,GAAG;AACpD,UAAM,IAAI,MAAM,wCAAwC,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EAC5E;AACA,MAAI,CAAC,oBAAoB,KAAK,IAAI,GAAG;AACnC,UAAM,IAAI,MAAM,+CAA+C,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EACnF;AACA,SAAO,EAAE,MAAM,KAAK;AACtB;AAEO,SAAS,sBAAsB,UAAkC,CAAC,GAAqB;AAC5F,MACE,QAAQ,cAAc,WAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,GACrE,OAAM,IAAI,MAAM,iDAAiD;AACnE,QAAM,aAAa,QAAQ,cAAc;AACzC,MAAI,CAAC,iBAAiB,IAAI,UAAU,EAAG,OAAM,IAAI,MAAM,+BAA+B;AACtF,MACE,QAAQ,8BAA8B,UACnC,OAAO,QAAQ,8BAA8B,WAChD,OAAM,IAAI,MAAM,yDAAyD;AAC3E,MAAI,kBAAkB;AACtB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWR,SAAS,MAAM;AACb,YAAM,EAAE,MAAM,KAAK,IAAI,YAAY,IAAI;AACvC,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,yBAAyB,WAAW,sBAAsB,IAAI,EAAE,CAAC,IAAI,WAAW,YAAY,CAAC,cAAc,WAAW,YAAY,CAAC,qBAAqB,WAAW,IAAI,CAAC;AAAA,MACrN;AAAA,IACF;AAAA,IACA,aAAa;AACX,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,qBAAqB,WAAW,YAAY,CAAC,uBAAuB,kBAAkB;AAAA,MACnI;AAAA,IACF;AAAA,IAEA,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAEJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,QAAQ,KAAK,SAAS;AAE5B,YAAM,SAAS;AAAA;AAAA;AAAA,oBAGD,KAAK,UAAU,OAAO,CAAC,WAAW,KAAK,UAAU,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAYtC,KAAK;AAAA;AAAA;AAAA;AAIpC,YAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;AACrF,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,OAAO,EAAE,WAAW;AAAA,UACxB;AAAA,UACA,QAAQ;AAAA,YACN,eAAe,EAAE,IAAI,MAAM,EAAE,WAAW;AAAA,YACxC;AAAA,YACA,uCAAuC,YAAY,2BAA2B,EAAE,WAAW,+BAA+B,YAAY;AAAA,YACtI;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,qBAAqB,EAAE;AAAA,UAClC,EAAE,KAAK,IAAI;AAAA,UACX,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK,UAAU;AAC5B,aAAO,OAAO,SAAS,WAAW,OAAO;AAAA,IAC3C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;AACxC,YAAM,UAAU,QAAQ,4BAA4B;AAAA,QAClD,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,iBAAiB,EAAE;AAAA,MACrB,CAAC;AACD,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,GAAI,QAAQ,cAAc,SAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;AAAA,QAC1E,GAAI,YAAY,SAAY,CAAC,IAAI,EAAE,QAAQ;AAAA;AAAA;AAAA,QAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;AAAA,QACzD,MAAM,MAAM,KAAK;AACf,gBAAM;AAAA,YACJ,KAAK,KAAK,YAAY;AAAA,YACtB,KAAK,UAAU;AAAA,cACb,EAAE,aAAa,KAAK,IAAI,oBAAoB,uBAAuB,aAAa,SAAS;AAAA,YAC3F,CAAC;AAAA,UACH;AAAA,QACF;AAAA;AAAA;AAAA,QAGA,MAAM,CAAC,QAAQ,kBAAkB;AAAA,UAC/B,iBAAiB,KAAK,KAAK,YAAY;AAAA,UACvC;AAAA,UACA,YAAY,KAAK;AAAA,UACjB;AAAA,UACA,WAAW,gBAAgB;AAAA,QAC7B,CAAC;AAAA,QACD,MAAM,YAAY,KAAK;AAErB,gBAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,KAAK,OAAO,CAAC;AAC7F,iBAAO,eAAe,KAAK,IAAI,MAAM;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-53UPUNBZ.js

@@ -1,325 +0,0 @@
-// src/benchmarks/cad-design.ts
-import { execFile } from "child_process";
-import { mkdtemp, readFile, writeFile } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-async function openscad(args, cwd) {
-  const { stdout, stderr } = await execFileAsync("xvfb-run", ["-a", "openscad", ...args], {
-    cwd,
-    maxBuffer: 1024 * 1024 * 64,
-    timeout: 12e4
-  });
-  return { stdout, stderr };
-}
-function measureStl(stl, volumesFromStderr) {
-  const verts = [];
-  const tris = [];
-  const re = /vertex\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)/g;
-  let m;
-  const flat = [];
-  while (m = re.exec(stl)) flat.push([Number(m[1]), Number(m[2]), Number(m[3])]);
-  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i], flat[i + 1], flat[i + 2]]);
-  for (const v of flat) verts.push(v);
-  const xs = verts.map((v) => v[0]);
-  const ys = verts.map((v) => v[1]);
-  const zs = verts.map((v) => v[2]);
-  const min = (a) => Math.min(...a);
-  const max = (a) => Math.max(...a);
-  const minZ = min(zs);
-  const maxZ = max(zs);
-  const h = maxZ - minZ || 1;
-  const footprint = (lo, hi) => {
-    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h);
-    if (band.length < 3) return 0;
-    const bx = band.map((v) => v[0]);
-    const by = band.map((v) => v[1]);
-    return (max(bx) - min(bx)) * (max(by) - min(by));
-  };
-  let vol6 = 0;
-  for (const [a, b, c] of tris) {
-    vol6 += a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0]);
-  }
-  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ };
-  return {
-    triangles: tris.length,
-    volumes: volumesFromStderr,
-    bbox,
-    baseFootprint: footprint(0, 0.6),
-    topFootprint: footprint(0.75, 1),
-    solidVolume: Math.abs(vol6) / 6,
-    bboxVolume: bbox.x * bbox.y * bbox.z || 1
-  };
-}
-function inRange(v, [lo, hi]) {
-  return v >= lo && v <= hi;
-}
-function scoreGeometry(g, spec) {
-  const checks = {};
-  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes);
-  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles;
-  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x);
-  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y);
-  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z);
-  if (spec.pitchedRoof != null)
-    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint;
-  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume;
-  const vals = Object.values(checks);
-  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0;
-  return { checks, score };
-}
-function buildingGold(w, d, sh, ns, rh, t = 3) {
-  return `
-w=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};
-module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
-module roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);
-  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);
-  translate([0,0,ns*sh]) roof(w,d,rh);
-  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);
-  translate([w/2-7,-1,2]) cube([14,t+2,20]);
-}
-`.trim();
-}
-function buildingTask(o) {
-  const { id, desc, w, d, sh, ns, rh } = o;
-  const totalWall = ns * sh;
-  const totalH = totalWall + rh;
-  const r = Math.round;
-  const spec = {
-    volumes: [1, ns * 4 + 6],
-    bbox: {
-      x: [r(w * 0.82), r(w * 1.18)],
-      y: [r(d * 0.82), r((d + 1) * 1.2)],
-      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)]
-    },
-    minTriangles: 50,
-    pitchedRoof: 0.55,
-    hollowBelow: 0.72
-  };
-  const prompt = [
-    `Write OpenSCAD source for ${desc}.`,
-    "Requirements:",
-    `- footprint roughly ${w} (X) by ${d} (Y) units`,
-    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,
-    "- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)",
-    "- hollow shell (walls with an interior cavity), not a solid block",
-    "- a door opening and several windows",
-    "Output ONLY the .scad source, no prose, no code fences."
-  ].join("\n");
-  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } };
-}
-var BUILDING_CONFIGS = [
-  { id: "cottage", desc: "a single-story cottage", w: 50, d: 40, sh: 30, ns: 1, rh: 18 },
-  { id: "cabin", desc: "a small one-room cabin", w: 44, d: 36, sh: 26, ns: 1, rh: 22 },
-  { id: "bungalow", desc: "a wide single-story bungalow", w: 70, d: 52, sh: 28, ns: 1, rh: 20 },
-  { id: "barn", desc: "a long deep barn", w: 60, d: 96, sh: 42, ns: 1, rh: 26 },
-  { id: "warehouse", desc: "a large warehouse", w: 110, d: 80, sh: 40, ns: 1, rh: 26 },
-  { id: "chapel", desc: "a narrow tall chapel with a steep roof", w: 40, d: 72, sh: 48, ns: 1, rh: 34 },
-  { id: "farmhouse", desc: "a two-story farmhouse", w: 72, d: 56, sh: 30, ns: 2, rh: 26 },
-  { id: "two-story-manor", desc: "a wide two-story manor", w: 96, d: 64, sh: 32, ns: 2, rh: 28 },
-  { id: "townhouse", desc: "a narrow three-story townhouse", w: 42, d: 52, sh: 30, ns: 3, rh: 34 },
-  { id: "watchtower", desc: "a tall narrow three-story watchtower", w: 34, d: 34, sh: 30, ns: 3, rh: 32 }
-];
-var WATER_TOWER_GOLD = `
-$fn=48; r=24; bh=72; t=3; ch=26;
-union(){
-  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }
-  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);
-}
-`.trim();
-var AFRAME_GOLD = `
-w=44; d=60; h=66; t=3;
-module tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }
-  translate([w/2-7,-1,0]) cube([14,t+2,24]);
-}
-`.trim();
-var TASKS = [
-  {
-    id: "two-story-house",
-    prompt: [
-      "Write OpenSCAD source for a two-story house.",
-      "Requirements:",
-      "- footprint roughly 80 (X) by 60 (Y) units",
-      "- two stories, total wall height ~68 units",
-      "- a PITCHED / gabled roof on top (NOT a flat roof) \u2014 the roof must taper toward a ridge",
-      "- hollow shell (walls with an interior cavity), not a solid block",
-      "- at least one door opening and several windows",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 12],
-        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },
-        minTriangles: 60,
-        pitchedRoof: 0.55,
-        hollowBelow: 0.7
-      },
-      gold: `
-w=80; d=60; sh=34; t=3; rh=28;
-module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
-module roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);
-  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);
-  translate([0,0,2*sh]) roof(w,d,rh);
-  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);
-  translate([w/2-8,-1,2]) cube([16,t+2,22]);
-}
-`.trim()
-    }
-  },
-  ...BUILDING_CONFIGS.map(buildingTask),
-  {
-    id: "water-tower",
-    prompt: [
-      "Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.",
-      "Requirements:",
-      "- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity",
-      "- a CONICAL roof on top that tapers to a point (a pitched roof)",
-      "- wall thickness around 3 units",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 4],
-        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },
-        minTriangles: 60,
-        pitchedRoof: 0.55,
-        hollowBelow: 0.8
-      },
-      gold: WATER_TOWER_GOLD
-    }
-  },
-  {
-    id: "a-frame-cabin",
-    prompt: [
-      "Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.",
-      "Requirements:",
-      "- footprint roughly 44 (X) by 60 (Y) units",
-      "- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top",
-      "- hollow inside (an interior cavity), not a solid wedge",
-      "- a door opening at the front",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 4],
-        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },
-        minTriangles: 30,
-        pitchedRoof: 0.45,
-        hollowBelow: 0.8
-      },
-      gold: AFRAME_GOLD
-    }
-  },
-  {
-    id: "hex-planter",
-    prompt: [
-      "Write OpenSCAD source for a hexagonal planter pot.",
-      "Requirements:",
-      "- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45",
-      "- open at the top, closed at the bottom (a cavity for soil)",
-      "- a wall thickness around 3 units",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 3],
-        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },
-        minTriangles: 30,
-        hollowBelow: 0.7
-      },
-      gold: `
-$fn=6; or=25; h=45; t=3;
-difference(){
-  cylinder(h=h, r=or);
-  translate([0,0,t]) cylinder(h=h, r=or-t);
-}
-`.trim()
-    }
-  }
-];
-function createCadDesignAdapter() {
-  return {
-    name: "cad-design",
-    async preflight() {
-      try {
-        await execFileAsync("xvfb-run", ["-a", "openscad", "--version"], { timeout: 3e4 });
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `cad-design preflight failed: ${msg}
-Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). The judge runs \`xvfb-run -a openscad -o out.stl model.scad\` \u2014 both must be on PATH.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      let tasks = TASKS;
-      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
-      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
-      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: t.meta }));
-    },
-    async goldArtifact(task) {
-      const meta = task.metadata;
-      return meta?.gold;
-    },
-    async judge(task, artifact) {
-      const spec = task.metadata.spec;
-      const src = artifact.trim();
-      if (!src) return { resolved: false, score: 0, detail: "empty artifact" };
-      const dir = await mkdtemp(join(tmpdir(), "cad-"));
-      const scad = join(dir, "model.scad");
-      const stlPath = join(dir, "model.stl");
-      const pngPath = join(dir, "model.png");
-      await writeFile(scad, src);
-      let stderr = "";
-      try {
-        const r = await openscad(["-o", stlPath, scad], dir);
-        stderr = r.stderr;
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` };
-      }
-      try {
-        await openscad(
-          ["-o", pngPath, "--imgsize=1100,850", "--camera=40,30,40,55,0,25,260", "--colorscheme=Tomorrow", scad],
-          dir
-        );
-      } catch {
-      }
-      let stl;
-      try {
-        stl = await readFile(stlPath, "utf8");
-      } catch {
-        return { resolved: false, score: 0, detail: "compiled but produced no STL (empty geometry)" };
-      }
-      const volumes = Number(/Volumes:\s*(\d+)/.exec(stderr)?.[1] ?? "1");
-      const geo = measureStl(stl, volumes);
-      const { checks, score } = scoreGeometry(geo, spec);
-      const resolved = score === 1;
-      const detail = JSON.stringify({
-        checks,
-        geo: {
-          triangles: geo.triangles,
-          volumes: geo.volumes,
-          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },
-          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),
-          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null
-        },
-        renderPath: pngPath,
-        stlPath
-      });
-      return { resolved, score, detail };
-    }
-  };
-}
-
-export {
-  createCadDesignAdapter
-};
-//# sourceMappingURL=chunk-53UPUNBZ.js.map