@tangle-network/agent-bench 0.3.5 → 0.3.6

package/dist/chunk-53UPUNBZ.js

@@ -0,0 +1,325 @@
+// src/benchmarks/cad-design.ts
+import { execFile } from "child_process";
+import { mkdtemp, readFile, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+async function openscad(args, cwd) {
+  const { stdout, stderr } = await execFileAsync("xvfb-run", ["-a", "openscad", ...args], {
+    cwd,
+    maxBuffer: 1024 * 1024 * 64,
+    timeout: 12e4
+  });
+  return { stdout, stderr };
+}
+function measureStl(stl, volumesFromStderr) {
+  const verts = [];
+  const tris = [];
+  const re = /vertex\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)/g;
+  let m;
+  const flat = [];
+  while (m = re.exec(stl)) flat.push([Number(m[1]), Number(m[2]), Number(m[3])]);
+  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i], flat[i + 1], flat[i + 2]]);
+  for (const v of flat) verts.push(v);
+  const xs = verts.map((v) => v[0]);
+  const ys = verts.map((v) => v[1]);
+  const zs = verts.map((v) => v[2]);
+  const min = (a) => Math.min(...a);
+  const max = (a) => Math.max(...a);
+  const minZ = min(zs);
+  const maxZ = max(zs);
+  const h = maxZ - minZ || 1;
+  const footprint = (lo, hi) => {
+    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h);
+    if (band.length < 3) return 0;
+    const bx = band.map((v) => v[0]);
+    const by = band.map((v) => v[1]);
+    return (max(bx) - min(bx)) * (max(by) - min(by));
+  };
+  let vol6 = 0;
+  for (const [a, b, c] of tris) {
+    vol6 += a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0]);
+  }
+  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ };
+  return {
+    triangles: tris.length,
+    volumes: volumesFromStderr,
+    bbox,
+    baseFootprint: footprint(0, 0.6),
+    topFootprint: footprint(0.75, 1),
+    solidVolume: Math.abs(vol6) / 6,
+    bboxVolume: bbox.x * bbox.y * bbox.z || 1
+  };
+}
+function inRange(v, [lo, hi]) {
+  return v >= lo && v <= hi;
+}
+function scoreGeometry(g, spec) {
+  const checks = {};
+  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes);
+  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles;
+  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x);
+  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y);
+  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z);
+  if (spec.pitchedRoof != null)
+    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint;
+  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume;
+  const vals = Object.values(checks);
+  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0;
+  return { checks, score };
+}
+function buildingGold(w, d, sh, ns, rh, t = 3) {
+  return `
+w=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};
+module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
+module roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);
+  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);
+  translate([0,0,ns*sh]) roof(w,d,rh);
+  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);
+  translate([w/2-7,-1,2]) cube([14,t+2,20]);
+}
+`.trim();
+}
+function buildingTask(o) {
+  const { id, desc, w, d, sh, ns, rh } = o;
+  const totalWall = ns * sh;
+  const totalH = totalWall + rh;
+  const r = Math.round;
+  const spec = {
+    volumes: [1, ns * 4 + 6],
+    bbox: {
+      x: [r(w * 0.82), r(w * 1.18)],
+      y: [r(d * 0.82), r((d + 1) * 1.2)],
+      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)]
+    },
+    minTriangles: 50,
+    pitchedRoof: 0.55,
+    hollowBelow: 0.72
+  };
+  const prompt = [
+    `Write OpenSCAD source for ${desc}.`,
+    "Requirements:",
+    `- footprint roughly ${w} (X) by ${d} (Y) units`,
+    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,
+    "- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)",
+    "- hollow shell (walls with an interior cavity), not a solid block",
+    "- a door opening and several windows",
+    "Output ONLY the .scad source, no prose, no code fences."
+  ].join("\n");
+  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } };
+}
+var BUILDING_CONFIGS = [
+  { id: "cottage", desc: "a single-story cottage", w: 50, d: 40, sh: 30, ns: 1, rh: 18 },
+  { id: "cabin", desc: "a small one-room cabin", w: 44, d: 36, sh: 26, ns: 1, rh: 22 },
+  { id: "bungalow", desc: "a wide single-story bungalow", w: 70, d: 52, sh: 28, ns: 1, rh: 20 },
+  { id: "barn", desc: "a long deep barn", w: 60, d: 96, sh: 42, ns: 1, rh: 26 },
+  { id: "warehouse", desc: "a large warehouse", w: 110, d: 80, sh: 40, ns: 1, rh: 26 },
+  { id: "chapel", desc: "a narrow tall chapel with a steep roof", w: 40, d: 72, sh: 48, ns: 1, rh: 34 },
+  { id: "farmhouse", desc: "a two-story farmhouse", w: 72, d: 56, sh: 30, ns: 2, rh: 26 },
+  { id: "two-story-manor", desc: "a wide two-story manor", w: 96, d: 64, sh: 32, ns: 2, rh: 28 },
+  { id: "townhouse", desc: "a narrow three-story townhouse", w: 42, d: 52, sh: 30, ns: 3, rh: 34 },
+  { id: "watchtower", desc: "a tall narrow three-story watchtower", w: 34, d: 34, sh: 30, ns: 3, rh: 32 }
+];
+var WATER_TOWER_GOLD = `
+$fn=48; r=24; bh=72; t=3; ch=26;
+union(){
+  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }
+  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);
+}
+`.trim();
+var AFRAME_GOLD = `
+w=44; d=60; h=66; t=3;
+module tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }
+  translate([w/2-7,-1,0]) cube([14,t+2,24]);
+}
+`.trim();
+var TASKS = [
+  {
+    id: "two-story-house",
+    prompt: [
+      "Write OpenSCAD source for a two-story house.",
+      "Requirements:",
+      "- footprint roughly 80 (X) by 60 (Y) units",
+      "- two stories, total wall height ~68 units",
+      "- a PITCHED / gabled roof on top (NOT a flat roof) \u2014 the roof must taper toward a ridge",
+      "- hollow shell (walls with an interior cavity), not a solid block",
+      "- at least one door opening and several windows",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 12],
+        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },
+        minTriangles: 60,
+        pitchedRoof: 0.55,
+        hollowBelow: 0.7
+      },
+      gold: `
+w=80; d=60; sh=34; t=3; rh=28;
+module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
+module roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);
+  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);
+  translate([0,0,2*sh]) roof(w,d,rh);
+  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);
+  translate([w/2-8,-1,2]) cube([16,t+2,22]);
+}
+`.trim()
+    }
+  },
+  ...BUILDING_CONFIGS.map(buildingTask),
+  {
+    id: "water-tower",
+    prompt: [
+      "Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.",
+      "Requirements:",
+      "- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity",
+      "- a CONICAL roof on top that tapers to a point (a pitched roof)",
+      "- wall thickness around 3 units",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 4],
+        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },
+        minTriangles: 60,
+        pitchedRoof: 0.55,
+        hollowBelow: 0.8
+      },
+      gold: WATER_TOWER_GOLD
+    }
+  },
+  {
+    id: "a-frame-cabin",
+    prompt: [
+      "Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.",
+      "Requirements:",
+      "- footprint roughly 44 (X) by 60 (Y) units",
+      "- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top",
+      "- hollow inside (an interior cavity), not a solid wedge",
+      "- a door opening at the front",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 4],
+        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },
+        minTriangles: 30,
+        pitchedRoof: 0.45,
+        hollowBelow: 0.8
+      },
+      gold: AFRAME_GOLD
+    }
+  },
+  {
+    id: "hex-planter",
+    prompt: [
+      "Write OpenSCAD source for a hexagonal planter pot.",
+      "Requirements:",
+      "- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45",
+      "- open at the top, closed at the bottom (a cavity for soil)",
+      "- a wall thickness around 3 units",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 3],
+        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },
+        minTriangles: 30,
+        hollowBelow: 0.7
+      },
+      gold: `
+$fn=6; or=25; h=45; t=3;
+difference(){
+  cylinder(h=h, r=or);
+  translate([0,0,t]) cylinder(h=h, r=or-t);
+}
+`.trim()
+    }
+  }
+];
+function createCadDesignAdapter() {
+  return {
+    name: "cad-design",
+    async preflight() {
+      try {
+        await execFileAsync("xvfb-run", ["-a", "openscad", "--version"], { timeout: 3e4 });
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `cad-design preflight failed: ${msg}
+Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). The judge runs \`xvfb-run -a openscad -o out.stl model.scad\` \u2014 both must be on PATH.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      let tasks = TASKS;
+      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
+      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
+      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: t.meta }));
+    },
+    async goldArtifact(task) {
+      const meta = task.metadata;
+      return meta?.gold;
+    },
+    async judge(task, artifact) {
+      const spec = task.metadata.spec;
+      const src = artifact.trim();
+      if (!src) return { resolved: false, score: 0, detail: "empty artifact" };
+      const dir = await mkdtemp(join(tmpdir(), "cad-"));
+      const scad = join(dir, "model.scad");
+      const stlPath = join(dir, "model.stl");
+      const pngPath = join(dir, "model.png");
+      await writeFile(scad, src);
+      let stderr = "";
+      try {
+        const r = await openscad(["-o", stlPath, scad], dir);
+        stderr = r.stderr;
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` };
+      }
+      try {
+        await openscad(
+          ["-o", pngPath, "--imgsize=1100,850", "--camera=40,30,40,55,0,25,260", "--colorscheme=Tomorrow", scad],
+          dir
+        );
+      } catch {
+      }
+      let stl;
+      try {
+        stl = await readFile(stlPath, "utf8");
+      } catch {
+        return { resolved: false, score: 0, detail: "compiled but produced no STL (empty geometry)" };
+      }
+      const volumes = Number(/Volumes:\s*(\d+)/.exec(stderr)?.[1] ?? "1");
+      const geo = measureStl(stl, volumes);
+      const { checks, score } = scoreGeometry(geo, spec);
+      const resolved = score === 1;
+      const detail = JSON.stringify({
+        checks,
+        geo: {
+          triangles: geo.triangles,
+          volumes: geo.volumes,
+          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },
+          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),
+          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null
+        },
+        renderPath: pngPath,
+        stlPath
+      });
+      return { resolved, score, detail };
+    }
+  };
+}
+
+export {
+  createCadDesignAdapter
+};
+//# sourceMappingURL=chunk-53UPUNBZ.js.map

package/dist/chunk-53UPUNBZ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/cad-design.ts"],"sourcesContent":["/**\n * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the\n * REAL OpenSCAD engine: compile + export STL, then measure the produced solid\n * geometry against the task's spec. Fully deterministic — the authoritative CAD\n * kernel is the gate, not an LLM and not the agent's own claim.\n *\n * This is verifiable-reward CAD: an agent is far better at *writing* exact\n * parametric code than at clicking a GUI, and the kernel either produces the\n * specified geometry or it doesn't. Every judged artifact also renders a PNG,\n * so a refine loop's attempts become a watchable \"model getting better\" reel\n * (run-capsule consumes the renders).\n *\n * Spec checks are geometric and ungameable:\n *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)\n *   - volumes       : disconnected-solid count within [min,max]\n *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)\n *   - detail        : triangle count ≥ floor (not a degenerate single cube)\n *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat\n *                     box can't fake this; a real gabled/hipped roof tapers\n *   - hollow        : interior cavity present (walls, not a solid block) — the\n *                     bbox volume materially exceeds the printed solid volume\n *\n * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Spec assertions a CAD task can require. All are deterministic + geometric. */\nexport interface CadSpec {\n  volumes?: [number, number]\n  bbox?: { x?: [number, number]; y?: [number, number]; z?: [number, number] }\n  minTriangles?: number\n  /** Top-band XY footprint must be < this fraction of the base footprint. */\n  pitchedRoof?: number\n  /** Printed solid volume must be < this fraction of the bbox volume (hollow). */\n  hollowBelow?: number\n}\n\ninterface CadTaskMeta {\n  spec: CadSpec\n  /** A known-good .scad that satisfies the spec — the oracle for verify-judge. */\n  gold: string\n}\n\n/** Run openscad with xvfb (it needs a GL context even headless). */\nasync function openscad(args: string[], cwd: string): Promise<{ stdout: string; stderr: string }> {\n  // -a => auto-pick a free display; openscad writes Facets/Volumes to stderr.\n  const { stdout, stderr } = await execFileAsync('xvfb-run', ['-a', 'openscad', ...args], {\n    cwd,\n    maxBuffer: 1024 * 1024 * 64,\n    timeout: 120_000,\n  })\n  return { stdout, stderr }\n}\n\ninterface Geometry {\n  triangles: number\n  volumes: number\n  bbox: { x: number; y: number; z: number }\n  /** XY footprint area (extent_x * extent_y) of the bottom 60% and top 25% z-bands. */\n  baseFootprint: number\n  topFootprint: number\n  /** Convex-ish solid volume estimate from the STL (sum of signed tetra volumes). */\n  solidVolume: number\n  bboxVolume: number\n}\n\n/** Parse an ASCII STL into the geometric measures the spec checks. Pure. */\nfunction measureStl(stl: string, volumesFromStderr: number): Geometry {\n  const verts: Array<[number, number, number]> = []\n  const tris: Array<[[number, number, number], [number, number, number], [number, number, number]]> = []\n  const re = /vertex\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)/g\n  let m: RegExpExecArray | null\n  const flat: Array<[number, number, number]> = []\n  while ((m = re.exec(stl))) flat.push([Number(m[1]), Number(m[2]), Number(m[3])])\n  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i]!, flat[i + 1]!, flat[i + 2]!])\n  for (const v of flat) verts.push(v)\n\n  const xs = verts.map((v) => v[0])\n  const ys = verts.map((v) => v[1])\n  const zs = verts.map((v) => v[2])\n  const min = (a: number[]) => Math.min(...a)\n  const max = (a: number[]) => Math.max(...a)\n  const minZ = min(zs)\n  const maxZ = max(zs)\n  const h = maxZ - minZ || 1\n\n  // Footprint of a z-band = XY extent of the vertices within it.\n  const footprint = (lo: number, hi: number): number => {\n    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h)\n    if (band.length < 3) return 0\n    const bx = band.map((v) => v[0])\n    const by = band.map((v) => v[1])\n    return (max(bx) - min(bx)) * (max(by) - min(by))\n  }\n\n  // Signed volume via the divergence theorem over triangles (|Σ v0·(v1×v2)/6|).\n  let vol6 = 0\n  for (const [a, b, c] of tris) {\n    vol6 +=\n      a[0] * (b[1] * c[2] - b[2] * c[1]) -\n      a[1] * (b[0] * c[2] - b[2] * c[0]) +\n      a[2] * (b[0] * c[1] - b[1] * c[0])\n  }\n  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ }\n  return {\n    triangles: tris.length,\n    volumes: volumesFromStderr,\n    bbox,\n    baseFootprint: footprint(0, 0.6),\n    topFootprint: footprint(0.75, 1),\n    solidVolume: Math.abs(vol6) / 6,\n    bboxVolume: bbox.x * bbox.y * bbox.z || 1,\n  }\n}\n\nfunction inRange(v: number, [lo, hi]: [number, number]): boolean {\n  return v >= lo && v <= hi\n}\n\n/** Score the geometry against the spec — each check is a named pass/fail. */\nfunction scoreGeometry(g: Geometry, spec: CadSpec): { checks: Record<string, boolean>; score: number } {\n  const checks: Record<string, boolean> = {}\n  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes)\n  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles\n  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x)\n  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y)\n  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z)\n  if (spec.pitchedRoof != null)\n    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint\n  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume\n  const vals = Object.values(checks)\n  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0\n  return { checks, score }\n}\n\n/** A parametric multi-story gabled building — the gold generator for the\n *  building-family tasks. They share geometry (hollow stacked shells + a gabled\n *  roof that tapers to a ridge + cut openings) and differ only in dimensions, so\n *  one verified generator yields a spec-passing oracle for each. */\nfunction buildingGold(w: number, d: number, sh: number, ns: number, rh: number, t = 3): string {\n  return `\nw=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);\n  translate([0,0,ns*sh]) roof(w,d,rh);\n  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);\n  translate([w/2-7,-1,2]) cube([14,t+2,20]);\n}\n`.trim()\n}\n\n/** Build a building-family task (prompt + spec + gold) from its dimensions.\n *  The spec's bbox is derived from the dims with a ±~18% tolerance so the agent\n *  must respect the brief's size; the gold (buildingGold) lands exactly on the\n *  nominal dims, so it passes by construction. Keep rh > ns*sh/3 (else the roof\n *  base sits inside the top-25% band and pitchedRoof fails). */\nfunction buildingTask(o: { id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }): {\n  id: string\n  prompt: string\n  meta: CadTaskMeta\n} {\n  const { id, desc, w, d, sh, ns, rh } = o\n  const totalWall = ns * sh\n  const totalH = totalWall + rh\n  const r = Math.round\n  const spec: CadSpec = {\n    volumes: [1, ns * 4 + 6],\n    bbox: {\n      x: [r(w * 0.82), r(w * 1.18)],\n      y: [r(d * 0.82), r((d + 1) * 1.2)],\n      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)],\n    },\n    minTriangles: 50,\n    pitchedRoof: 0.55,\n    hollowBelow: 0.72,\n  }\n  const prompt = [\n    `Write OpenSCAD source for ${desc}.`,\n    'Requirements:',\n    `- footprint roughly ${w} (X) by ${d} (Y) units`,\n    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,\n    '- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)',\n    '- hollow shell (walls with an interior cavity), not a solid block',\n    '- a door opening and several windows',\n    'Output ONLY the .scad source, no prose, no code fences.',\n  ].join('\\n')\n  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } }\n}\n\n/** The building family — one parametric skill (hit the bbox + pitched roof +\n *  hollow shell), sampled across footprints, heights, story counts, roof rises. */\nconst BUILDING_CONFIGS: Array<{ id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }> = [\n  { id: 'cottage', desc: 'a single-story cottage', w: 50, d: 40, sh: 30, ns: 1, rh: 18 },\n  { id: 'cabin', desc: 'a small one-room cabin', w: 44, d: 36, sh: 26, ns: 1, rh: 22 },\n  { id: 'bungalow', desc: 'a wide single-story bungalow', w: 70, d: 52, sh: 28, ns: 1, rh: 20 },\n  { id: 'barn', desc: 'a long deep barn', w: 60, d: 96, sh: 42, ns: 1, rh: 26 },\n  { id: 'warehouse', desc: 'a large warehouse', w: 110, d: 80, sh: 40, ns: 1, rh: 26 },\n  { id: 'chapel', desc: 'a narrow tall chapel with a steep roof', w: 40, d: 72, sh: 48, ns: 1, rh: 34 },\n  { id: 'farmhouse', desc: 'a two-story farmhouse', w: 72, d: 56, sh: 30, ns: 2, rh: 26 },\n  { id: 'two-story-manor', desc: 'a wide two-story manor', w: 96, d: 64, sh: 32, ns: 2, rh: 28 },\n  { id: 'townhouse', desc: 'a narrow three-story townhouse', w: 42, d: 52, sh: 30, ns: 3, rh: 34 },\n  { id: 'watchtower', desc: 'a tall narrow three-story watchtower', w: 34, d: 34, sh: 30, ns: 3, rh: 32 },\n]\n\n/** A hollow cylindrical water tower with a conical roof — a non-box shape that\n *  still exercises pitched-roof (the cone tapers to an apex) + hollow + bbox. */\nconst WATER_TOWER_GOLD = `\n$fn=48; r=24; bh=72; t=3; ch=26;\nunion(){\n  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }\n  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);\n}\n`.trim()\n\n/** An A-frame: the steep roof IS the walls — a hollow triangular prism. The most\n *  extreme taper in the set; tests that the agent can build a non-rectangular shell. */\nconst AFRAME_GOLD = `\nw=44; d=60; h=66; t=3;\nmodule tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }\n  translate([w/2-7,-1,0]) cube([14,t+2,24]);\n}\n`.trim()\n\nconst TASKS: Array<{ id: string; prompt: string; meta: CadTaskMeta }> = [\n  {\n    id: 'two-story-house',\n    prompt: [\n      'Write OpenSCAD source for a two-story house.',\n      'Requirements:',\n      '- footprint roughly 80 (X) by 60 (Y) units',\n      '- two stories, total wall height ~68 units',\n      '- a PITCHED / gabled roof on top (NOT a flat roof) — the roof must taper toward a ridge',\n      '- hollow shell (walls with an interior cavity), not a solid block',\n      '- at least one door opening and several windows',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 12],\n        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.7,\n      },\n      gold: `\nw=80; d=60; sh=34; t=3; rh=28;\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);\n  translate([0,0,2*sh]) roof(w,d,rh);\n  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);\n  translate([w/2-8,-1,2]) cube([16,t+2,22]);\n}\n`.trim(),\n    },\n  },\n  ...BUILDING_CONFIGS.map(buildingTask),\n  {\n    id: 'water-tower',\n    prompt: [\n      'Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.',\n      'Requirements:',\n      '- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity',\n      '- a CONICAL roof on top that tapers to a point (a pitched roof)',\n      '- wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.8,\n      },\n      gold: WATER_TOWER_GOLD,\n    },\n  },\n  {\n    id: 'a-frame-cabin',\n    prompt: [\n      'Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.',\n      'Requirements:',\n      '- footprint roughly 44 (X) by 60 (Y) units',\n      '- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top',\n      '- hollow inside (an interior cavity), not a solid wedge',\n      '- a door opening at the front',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },\n        minTriangles: 30,\n        pitchedRoof: 0.45,\n        hollowBelow: 0.8,\n      },\n      gold: AFRAME_GOLD,\n    },\n  },\n  {\n    id: 'hex-planter',\n    prompt: [\n      'Write OpenSCAD source for a hexagonal planter pot.',\n      'Requirements:',\n      '- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45',\n      '- open at the top, closed at the bottom (a cavity for soil)',\n      '- a wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 3],\n        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },\n        minTriangles: 30,\n        hollowBelow: 0.7,\n      },\n      gold: `\n$fn=6; or=25; h=45; t=3;\ndifference(){\n  cylinder(h=h, r=or);\n  translate([0,0,t]) cylinder(h=h, r=or-t);\n}\n`.trim(),\n    },\n  },\n]\n\nexport function createCadDesignAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cad-design',\n\n    async preflight() {\n      try {\n        await execFileAsync('xvfb-run', ['-a', 'openscad', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `cad-design preflight failed: ${msg}\\n` +\n            `Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). ` +\n            `The judge runs \\`xvfb-run -a openscad -o out.stl model.scad\\` — both must be on PATH.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let tasks = TASKS\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: t.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = task.metadata as unknown as CadTaskMeta | undefined\n      return meta?.gold\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const spec = (task.metadata as unknown as CadTaskMeta).spec\n      const src = artifact.trim()\n      if (!src) return { resolved: false, score: 0, detail: 'empty artifact' }\n\n      const dir = await mkdtemp(join(tmpdir(), 'cad-'))\n      const scad = join(dir, 'model.scad')\n      const stlPath = join(dir, 'model.stl')\n      const pngPath = join(dir, 'model.png')\n      await writeFile(scad, src)\n\n      // GATE 1: compile + export STL. Nonzero exit / CGAL error => does not compile.\n      let stderr = ''\n      try {\n        const r = await openscad(['-o', stlPath, scad], dir)\n        stderr = r.stderr\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` }\n      }\n\n      // Render a PNG alongside (the artifact run-capsule animates). Non-fatal.\n      try {\n        await openscad(\n          ['-o', pngPath, '--imgsize=1100,850', '--camera=40,30,40,55,0,25,260', '--colorscheme=Tomorrow', scad],\n          dir,\n        )\n      } catch {\n        /* render is for the reel; geometry gate already has the STL */\n      }\n\n      let stl: string\n      try {\n        stl = await readFile(stlPath, 'utf8')\n      } catch {\n        return { resolved: false, score: 0, detail: 'compiled but produced no STL (empty geometry)' }\n      }\n      const volumes = Number(/Volumes:\\s*(\\d+)/.exec(stderr)?.[1] ?? '1')\n      const geo = measureStl(stl, volumes)\n      const { checks, score } = scoreGeometry(geo, spec)\n      const resolved = score === 1\n      const detail = JSON.stringify({\n        checks,\n        geo: {\n          triangles: geo.triangles,\n          volumes: geo.volumes,\n          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },\n          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),\n          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null,\n        },\n        renderPath: pngPath,\n        stlPath,\n      })\n      return { resolved, score, detail }\n    },\n  }\n}\n"],"mappings":";AAyBA,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,iBAAiB;AAC7C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AAoBxC,eAAe,SAAS,MAAgB,KAA0D;AAEhG,QAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,YAAY,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG;AAAA,IACtF;AAAA,IACA,WAAW,OAAO,OAAO;AAAA,IACzB,SAAS;AAAA,EACX,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO;AAC1B;AAeA,SAAS,WAAW,KAAa,mBAAqC;AACpE,QAAM,QAAyC,CAAC;AAChD,QAAM,OAA8F,CAAC;AACrG,QAAM,KAAK;AACX,MAAI;AACJ,QAAM,OAAwC,CAAC;AAC/C,SAAQ,IAAI,GAAG,KAAK,GAAG,EAAI,MAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/E,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK,QAAQ,KAAK,EAAG,MAAK,KAAK,CAAC,KAAK,CAAC,GAAI,KAAK,IAAI,CAAC,GAAI,KAAK,IAAI,CAAC,CAAE,CAAC;AAC7F,aAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAElC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,IAAI,OAAO,QAAQ;AAGzB,QAAM,YAAY,CAAC,IAAY,OAAuB;AACpD,UAAM,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,KAAK,EAAE,CAAC,KAAK,OAAO,KAAK,CAAC;AAC/E,QAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,YAAQ,IAAI,EAAE,IAAI,IAAI,EAAE,MAAM,IAAI,EAAE,IAAI,IAAI,EAAE;AAAA,EAChD;AAGA,MAAI,OAAO;AACX,aAAW,CAAC,GAAG,GAAG,CAAC,KAAK,MAAM;AAC5B,YACE,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpC;AACA,QAAM,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,OAAO,KAAK;AAC1E,SAAO;AAAA,IACL,WAAW,KAAK;AAAA,IAChB,SAAS;AAAA,IACT;AAAA,IACA,eAAe,UAAU,GAAG,GAAG;AAAA,IAC/B,cAAc,UAAU,MAAM,CAAC;AAAA,IAC/B,aAAa,KAAK,IAAI,IAAI,IAAI;AAAA,IAC9B,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,KAAK;AAAA,EAC1C;AACF;AAEA,SAAS,QAAQ,GAAW,CAAC,IAAI,EAAE,GAA8B;AAC/D,SAAO,KAAK,MAAM,KAAK;AACzB;AAGA,SAAS,cAAc,GAAa,MAAmE;AACrG,QAAM,SAAkC,CAAC;AACzC,MAAI,KAAK,QAAS,QAAO,UAAU,QAAQ,EAAE,SAAS,KAAK,OAAO;AAClE,MAAI,KAAK,gBAAgB,KAAM,QAAO,SAAS,EAAE,aAAa,KAAK;AACnE,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,eAAe;AACtB,WAAO,cAAc,EAAE,gBAAgB,KAAK,EAAE,eAAe,KAAK,cAAc,EAAE;AACpF,MAAI,KAAK,eAAe,KAAM,QAAO,SAAS,EAAE,cAAc,KAAK,cAAc,EAAE;AACnF,QAAM,OAAO,OAAO,OAAO,MAAM;AACjC,QAAM,QAAQ,KAAK,SAAS,KAAK,OAAO,OAAO,EAAE,SAAS,KAAK,SAAS;AACxE,SAAO,EAAE,QAAQ,MAAM;AACzB;AAMA,SAAS,aAAa,GAAW,GAAW,IAAY,IAAY,IAAY,IAAI,GAAW;AAC7F,SAAO;AAAA,IACL,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUjD,KAAK;AACP;AAOA,SAAS,aAAa,GAIpB;AACA,QAAM,EAAE,IAAI,MAAM,GAAG,GAAG,IAAI,IAAI,GAAG,IAAI;AACvC,QAAM,YAAY,KAAK;AACvB,QAAM,SAAS,YAAY;AAC3B,QAAM,IAAI,KAAK;AACf,QAAM,OAAgB;AAAA,IACpB,SAAS,CAAC,GAAG,KAAK,IAAI,CAAC;AAAA,IACvB,MAAM;AAAA,MACJ,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,CAAC;AAAA,MAC5B,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC;AAAA,MACjC,GAAG,CAAC,EAAE,SAAS,IAAI,GAAG,GAAG,SAAS,KAAK,GAAG,CAAC;AAAA,IAC7C;AAAA,IACA,cAAc;AAAA,IACd,aAAa;AAAA,IACb,aAAa;AAAA,EACf;AACA,QAAM,SAAS;AAAA,IACb,6BAA6B,IAAI;AAAA,IACjC;AAAA,IACA,uBAAuB,CAAC,WAAW,CAAC;AAAA,IACpC,KAAK,IAAI,KAAK,EAAE,gCAAgC,SAAS,WAAW,6BAA6B,EAAE;AAAA,IACnG;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACX,SAAO,EAAE,IAAI,QAAQ,MAAM,EAAE,MAAM,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE;AAC5E;AAIA,IAAM,mBAAkH;AAAA,EACtH,EAAE,IAAI,WAAW,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACrF,EAAE,IAAI,SAAS,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,YAAY,MAAM,gCAAgC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5F,EAAE,IAAI,QAAQ,MAAM,oBAAoB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5E,EAAE,IAAI,aAAa,MAAM,qBAAqB,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,UAAU,MAAM,0CAA0C,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACpG,EAAE,IAAI,aAAa,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACtF,EAAE,IAAI,mBAAmB,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC7F,EAAE,IAAI,aAAa,MAAM,kCAAkC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC/F,EAAE,IAAI,cAAc,MAAM,wCAAwC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AACxG;AAIA,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMvB,KAAK;AAIP,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOlB,KAAK;AAEP,IAAM,QAAkE;AAAA,EACtE;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,EAAE;AAAA,QACf,MAAM,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAChD,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWV,KAAK;AAAA,IACH;AAAA,EACF;AAAA,EACA,GAAG,iBAAiB,IAAI,YAAY;AAAA,EACpC;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAC/C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMV,KAAK;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM,cAAc,YAAY,CAAC,MAAM,YAAY,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MACtF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,gCAAgC,GAAG;AAAA;AAAA,QAGrC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,QAAQ;AACZ,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,EAAE,KAA2C,EAAE;AAAA,IAC7H;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK;AAClB,aAAO,MAAM;AAAA,IACf;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAQ,KAAK,SAAoC;AACvD,YAAM,MAAM,SAAS,KAAK;AAC1B,UAAI,CAAC,IAAK,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AAEvE,YAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,MAAM,CAAC;AAChD,YAAM,OAAO,KAAK,KAAK,YAAY;AACnC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,MAAM,GAAG;AAGzB,UAAI,SAAS;AACb,UAAI;AACF,cAAM,IAAI,MAAM,SAAS,CAAC,MAAM,SAAS,IAAI,GAAG,GAAG;AACnD,iBAAS,EAAE;AAAA,MACb,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,MACrF;AAGA,UAAI;AACF,cAAM;AAAA,UACJ,CAAC,MAAM,SAAS,sBAAsB,iCAAiC,0BAA0B,IAAI;AAAA,UACrG;AAAA,QACF;AAAA,MACF,QAAQ;AAAA,MAER;AAEA,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,SAAS,SAAS,MAAM;AAAA,MACtC,QAAQ;AACN,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gDAAgD;AAAA,MAC9F;AACA,YAAM,UAAU,OAAO,mBAAmB,KAAK,MAAM,IAAI,CAAC,KAAK,GAAG;AAClE,YAAM,MAAM,WAAW,KAAK,OAAO;AACnC,YAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,KAAK,IAAI;AACjD,YAAM,WAAW,UAAU;AAC3B,YAAM,SAAS,KAAK,UAAU;AAAA,QAC5B;AAAA,QACA,KAAK;AAAA,UACH,WAAW,IAAI;AAAA,UACf,SAAS,IAAI;AAAA,UACb,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,EAAE;AAAA,UACxF,aAAa,EAAE,IAAI,cAAc,IAAI,YAAY,QAAQ,CAAC;AAAA,UAC1D,oBAAoB,IAAI,gBAAgB,EAAE,IAAI,eAAe,IAAI,eAAe,QAAQ,CAAC,IAAI;AAAA,QAC/F;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,MACF,CAAC;AACD,aAAO,EAAE,UAAU,OAAO,OAAO;AAAA,IACnC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-5SBJCB6W.js

@@ -0,0 +1,144 @@
+import {
+  createToolLlmAdapter
+} from "./chunk-2PVVP7GN.js";
+import {
+  createTrataHedgeAdapter
+} from "./chunk-VQRS7VUC.js";
+import {
+  createWebArenaVerifiedAdapter
+} from "./chunk-Z7ML6L77.js";
+import {
+  createRagBenchAdapter
+} from "./chunk-R67DFVLO.js";
+import {
+  createSimpleQaAdapter
+} from "./chunk-SEVJPLZC.js";
+import {
+  createSweBenchAdapter
+} from "./chunk-CKUVRZ2T.js";
+import {
+  createT2RagBenchAdapter
+} from "./chunk-Y6O2OCUO.js";
+import {
+  createTau2BenchAdapter
+} from "./chunk-HBSWHQNJ.js";
+import {
+  createTau3BankingAdapter
+} from "./chunk-UAIOHCUK.js";
+import {
+  createTerminalBenchAdapter
+} from "./chunk-JRWWGMK7.js";
+import {
+  createFramesAdapter
+} from "./chunk-INNOYXCP.js";
+import {
+  createHotpotqaAdapter
+} from "./chunk-PA2ZKHJC.js";
+import {
+  createHumanEvalAdapter
+} from "./chunk-PPYSEKFM.js";
+import {
+  createMind2WebAdapter
+} from "./chunk-TBKU5XQI.js";
+import {
+  createNoMiraclAdapter
+} from "./chunk-TSWPNOYM.js";
+import {
+  createOpenRagBenchAdapter
+} from "./chunk-KDIKRJGB.js";
+import {
+  createProgrambenchAdapter
+} from "./chunk-ZEWMTR5M.js";
+import {
+  createCadBenchAdapter
+} from "./chunk-PUIRNYI7.js";
+import {
+  createCadGenBenchAdapter
+} from "./chunk-C7T7WEK2.js";
+import {
+  createCommit0Adapter
+} from "./chunk-2XU6OGEN.js";
+import {
+  createCragAdapter
+} from "./chunk-HHXFIHXC.js";
+import {
+  createDabstepAdapter
+} from "./chunk-J3KDJNX2.js";
+import {
+  createEnterpriseOpsGymAdapter
+} from "./chunk-X5YKXC6V.js";
+import {
+  createFinResearchBenchAdapter
+} from "./chunk-YCGY7UIZ.js";
+import {
+  createFinsearchcompAdapter
+} from "./chunk-UPAMRDX4.js";
+import {
+  createAecBenchAdapter
+} from "./chunk-ODT47UAY.js";
+import {
+  createAgentBenchAdapter
+} from "./chunk-7WSD27QQ.js";
+import {
+  createAppWorldAdapter,
+  createAppWorldReactAdapter
+} from "./chunk-SYDW647C.js";
+import {
+  createBfclAdapter
+} from "./chunk-IFAV6KEM.js";
+import {
+  createCadDesignAdapter
+} from "./chunk-53UPUNBZ.js";
+
+// src/adapters.ts
+var ADAPTERS = {
+  "swe-bench": createSweBenchAdapter,
+  "terminal-bench": createTerminalBenchAdapter,
+  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).
+  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge
+  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.
+  "aec-bench": createAecBenchAdapter,
+  commit0: createCommit0Adapter,
+  dabstep: createDabstepAdapter,
+  programbench: createProgrambenchAdapter,
+  "webarena-verified": createWebArenaVerifiedAdapter,
+  "tau2-bench": createTau2BenchAdapter,
+  "tau3-banking": createTau3BankingAdapter,
+  agentbench: createAgentBenchAdapter,
+  bfcl: createBfclAdapter,
+  toollm: createToolLlmAdapter,
+  appworld: createAppWorldAdapter,
+  // AppWorld's native interactive protocol — the worker is the in-engine ReAct
+  // episode (execution feedback every turn), the mode published baselines use.
+  "appworld-react": createAppWorldReactAdapter,
+  "enterpriseops-gym": createEnterpriseOpsGymAdapter,
+  "cad-design": createCadDesignAdapter,
+  cadbench: createCadBenchAdapter,
+  cadgenbench: createCadGenBenchAdapter,
+  frames: createFramesAdapter,
+  ragbench: createRagBenchAdapter,
+  crag: createCragAdapter,
+  nomiracl: createNoMiraclAdapter,
+  "open-rag-bench": createOpenRagBenchAdapter,
+  "t2-ragbench": createT2RagBenchAdapter,
+  finresearchbench: createFinResearchBenchAdapter,
+  finsearchcomp: createFinsearchcompAdapter,
+  simpleqa: createSimpleQaAdapter,
+  hotpotqa: createHotpotqaAdapter,
+  // Deployable-checker code domain: worker = router completion, judge = Docker test
+  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).
+  humaneval: createHumanEvalAdapter,
+  mind2web: createMind2WebAdapter,
+  "trata-hedge": createTrataHedgeAdapter
+};
+function resolveAdapter(key) {
+  const make = ADAPTERS[key];
+  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(", ")})`);
+  return make();
+}
+
+export {
+  ADAPTERS,
+  resolveAdapter
+};
+//# sourceMappingURL=chunk-5SBJCB6W.js.map

package/dist/chunk-5SBJCB6W.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/adapters.ts"],"sourcesContent":["/**\n * The benchmark registry — the single source of truth for every wired benchmark.\n * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`\n * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it\n * here rather than each keeping their own copy.\n */\n\nimport { createAecBenchAdapter } from './benchmarks/aec-bench'\nimport { createAgentBenchAdapter } from './benchmarks/agentbench'\nimport { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'\nimport { createBfclAdapter } from './benchmarks/bfcl'\nimport { createCadBenchAdapter } from './benchmarks/cadbench'\nimport { createCadDesignAdapter } from './benchmarks/cad-design'\nimport { createCadGenBenchAdapter } from './benchmarks/cadgenbench'\nimport { createCommit0Adapter } from './benchmarks/commit0'\nimport { createCragAdapter } from './benchmarks/crag'\nimport { createDabstepAdapter } from './benchmarks/dabstep'\nimport { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'\nimport { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'\nimport { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'\nimport { createFramesAdapter } from './benchmarks/frames'\nimport { createHotpotqaAdapter } from './benchmarks/hotpotqa'\nimport { createHumanEvalAdapter } from './benchmarks/humaneval'\nimport { createMind2WebAdapter } from './benchmarks/mind2web'\nimport { createNoMiraclAdapter } from './benchmarks/nomiracl'\nimport { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'\nimport { createProgrambenchAdapter } from './benchmarks/programbench'\nimport { createRagBenchAdapter } from './benchmarks/ragbench'\nimport { createSimpleQaAdapter } from './benchmarks/simpleqa'\nimport { createSweBenchAdapter } from './benchmarks/swe-bench'\nimport { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'\nimport { createTau2BenchAdapter } from './benchmarks/tau2-bench'\nimport { createTau3BankingAdapter } from './benchmarks/tau3-banking'\nimport { createTerminalBenchAdapter } from './benchmarks/terminal-bench'\nimport { createToolLlmAdapter } from './benchmarks/toollm'\nimport { createTrataHedgeAdapter } from './benchmarks/trata-hedge'\nimport { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'\nimport type { BenchmarkAdapter } from './benchmarks/types'\n\nexport const ADAPTERS: Record<string, () => BenchmarkAdapter> = {\n  'swe-bench': createSweBenchAdapter,\n  'terminal-bench': createTerminalBenchAdapter,\n  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).\n  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge\n  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.\n  'aec-bench': createAecBenchAdapter,\n  commit0: createCommit0Adapter,\n  dabstep: createDabstepAdapter,\n  programbench: createProgrambenchAdapter,\n  'webarena-verified': createWebArenaVerifiedAdapter,\n  'tau2-bench': createTau2BenchAdapter,\n  'tau3-banking': createTau3BankingAdapter,\n  agentbench: createAgentBenchAdapter,\n  bfcl: createBfclAdapter,\n  toollm: createToolLlmAdapter,\n  appworld: createAppWorldAdapter,\n  // AppWorld's native interactive protocol — the worker is the in-engine ReAct\n  // episode (execution feedback every turn), the mode published baselines use.\n  'appworld-react': createAppWorldReactAdapter,\n  'enterpriseops-gym': createEnterpriseOpsGymAdapter,\n  'cad-design': createCadDesignAdapter,\n  cadbench: createCadBenchAdapter,\n  cadgenbench: createCadGenBenchAdapter,\n  frames: createFramesAdapter,\n  ragbench: createRagBenchAdapter,\n  crag: createCragAdapter,\n  nomiracl: createNoMiraclAdapter,\n  'open-rag-bench': createOpenRagBenchAdapter,\n  't2-ragbench': createT2RagBenchAdapter,\n  finresearchbench: createFinResearchBenchAdapter,\n  finsearchcomp: createFinsearchcompAdapter,\n  simpleqa: createSimpleQaAdapter,\n  hotpotqa: createHotpotqaAdapter,\n  // Deployable-checker code domain: worker = router completion, judge = Docker test\n  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).\n  humaneval: createHumanEvalAdapter,\n  mind2web: createMind2WebAdapter,\n  'trata-hedge': createTrataHedgeAdapter,\n}\n\n/** Resolve a benchmark key to its adapter, failing loud with the known keys. */\nexport function resolveAdapter(key: string): BenchmarkAdapter {\n  const make = ADAPTERS[key]\n  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)\n  return make()\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCO,IAAM,WAAmD;AAAA,EAC9D,aAAa;AAAA,EACb,kBAAkB;AAAA;AAAA;AAAA;AAAA,EAIlB,aAAa;AAAA,EACb,SAAS;AAAA,EACT,SAAS;AAAA,EACT,cAAc;AAAA,EACd,qBAAqB;AAAA,EACrB,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,UAAU;AAAA;AAAA;AAAA,EAGV,kBAAkB;AAAA,EAClB,qBAAqB;AAAA,EACrB,cAAc;AAAA,EACd,UAAU;AAAA,EACV,aAAa;AAAA,EACb,QAAQ;AAAA,EACR,UAAU;AAAA,EACV,MAAM;AAAA,EACN,UAAU;AAAA,EACV,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,UAAU;AAAA,EACV,UAAU;AAAA;AAAA;AAAA,EAGV,WAAW;AAAA,EACX,UAAU;AAAA,EACV,eAAe;AACjB;AAGO,SAAS,eAAe,KAA+B;AAC5D,QAAM,OAAO,SAAS,GAAG;AACzB,MAAI,CAAC,KAAM,OAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,GAAG,CAAC,WAAW,OAAO,KAAK,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG;AACjH,SAAO,KAAK;AACd;","names":[]}

package/dist/chunk-7WSD27QQ.js

@@ -0,0 +1,118 @@
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/agentbench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "agentbench-dbbench.json");
+var DEFAULT_SPLIT = "dev";
+var agentbenchDir = () => process.env.AGENTBENCH_DIR;
+var agentbenchAnswerOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+function normalizeAnswer(value) {
+  return value.toLowerCase().replace(/\s+/g, " ").replace(/^["'`]+|["'`]+$/g, "").trim();
+}
+function rowToTask(row, index, split) {
+  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ""}`).join(", ");
+  const sampleRows = row.table?.table_info?.rows?.slice(0, 40);
+  const meta = {
+    labels: row.label,
+    split,
+    subset: "dbbench",
+    table: row.table
+  };
+  return {
+    id: `dbbench-${split}-${index}`,
+    split,
+    prompt: [
+      "Answer this AgentBench DBBench question using the table below.",
+      "Return only the answer value.",
+      "",
+      `Question: ${row.description}`,
+      row.table?.table_name ? `Table: ${row.table.table_name}` : void 0,
+      columns ? `Columns: ${columns}` : void 0,
+      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.labels)) {
+    throw new Error(`agentbench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts, split) {
+  let tasks = rows.map((row, index) => rowToTask(row, index, split));
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+async function loadJsonl(path) {
+  const raw = await readFile(path, "utf8");
+  return raw.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
+}
+async function loadFixtures(opts, split) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 \u2014 loading ${rows.length} DBBench adapter fixtures`);
+  return selectRows(rows, opts, split);
+}
+function createAgentBenchAdapter() {
+  const fixturesMode = process.env.AGENTBENCH_FIXTURES === "1";
+  return {
+    name: "agentbench",
+    output: agentbenchAnswerOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = agentbenchDir();
+      if (!dir) {
+        throw new Error("AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.");
+      }
+      await loadJsonl(join(dir, "data", "dbbench", `${DEFAULT_SPLIT}.jsonl`));
+    },
+    async loadTasks(opts = {}) {
+      const split = opts.split ?? DEFAULT_SPLIT;
+      if (fixturesMode) return loadFixtures(opts, split);
+      const dir = agentbenchDir();
+      if (!dir) throw new Error("AGENTBENCH_DIR is required to load AgentBench DBBench tasks");
+      return selectRows(await loadJsonl(join(dir, "data", "dbbench", `${split}.jsonl`)), opts, split);
+    },
+    async goldArtifact(task) {
+      return readMeta(task).labels[0];
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const got = normalizeAnswer(artifact);
+      const expected = meta.labels.map(normalizeAnswer);
+      const resolved = expected.includes(got);
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact })
+      };
+    }
+  };
+}
+
+export {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+};
+//# sourceMappingURL=chunk-7WSD27QQ.js.map