@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/chunk-53UPUNBZ.js

@@ -0,0 +1,325 @@
+// src/benchmarks/cad-design.ts
+import { execFile } from "child_process";
+import { mkdtemp, readFile, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+async function openscad(args, cwd) {
+  const { stdout, stderr } = await execFileAsync("xvfb-run", ["-a", "openscad", ...args], {
+    cwd,
+    maxBuffer: 1024 * 1024 * 64,
+    timeout: 12e4
+  });
+  return { stdout, stderr };
+}
+function measureStl(stl, volumesFromStderr) {
+  const verts = [];
+  const tris = [];
+  const re = /vertex\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)/g;
+  let m;
+  const flat = [];
+  while (m = re.exec(stl)) flat.push([Number(m[1]), Number(m[2]), Number(m[3])]);
+  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i], flat[i + 1], flat[i + 2]]);
+  for (const v of flat) verts.push(v);
+  const xs = verts.map((v) => v[0]);
+  const ys = verts.map((v) => v[1]);
+  const zs = verts.map((v) => v[2]);
+  const min = (a) => Math.min(...a);
+  const max = (a) => Math.max(...a);
+  const minZ = min(zs);
+  const maxZ = max(zs);
+  const h = maxZ - minZ || 1;
+  const footprint = (lo, hi) => {
+    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h);
+    if (band.length < 3) return 0;
+    const bx = band.map((v) => v[0]);
+    const by = band.map((v) => v[1]);
+    return (max(bx) - min(bx)) * (max(by) - min(by));
+  };
+  let vol6 = 0;
+  for (const [a, b, c] of tris) {
+    vol6 += a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0]);
+  }
+  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ };
+  return {
+    triangles: tris.length,
+    volumes: volumesFromStderr,
+    bbox,
+    baseFootprint: footprint(0, 0.6),
+    topFootprint: footprint(0.75, 1),
+    solidVolume: Math.abs(vol6) / 6,
+    bboxVolume: bbox.x * bbox.y * bbox.z || 1
+  };
+}
+function inRange(v, [lo, hi]) {
+  return v >= lo && v <= hi;
+}
+function scoreGeometry(g, spec) {
+  const checks = {};
+  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes);
+  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles;
+  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x);
+  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y);
+  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z);
+  if (spec.pitchedRoof != null)
+    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint;
+  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume;
+  const vals = Object.values(checks);
+  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0;
+  return { checks, score };
+}
+function buildingGold(w, d, sh, ns, rh, t = 3) {
+  return `
+w=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};
+module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
+module roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);
+  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);
+  translate([0,0,ns*sh]) roof(w,d,rh);
+  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);
+  translate([w/2-7,-1,2]) cube([14,t+2,20]);
+}
+`.trim();
+}
+function buildingTask(o) {
+  const { id, desc, w, d, sh, ns, rh } = o;
+  const totalWall = ns * sh;
+  const totalH = totalWall + rh;
+  const r = Math.round;
+  const spec = {
+    volumes: [1, ns * 4 + 6],
+    bbox: {
+      x: [r(w * 0.82), r(w * 1.18)],
+      y: [r(d * 0.82), r((d + 1) * 1.2)],
+      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)]
+    },
+    minTriangles: 50,
+    pitchedRoof: 0.55,
+    hollowBelow: 0.72
+  };
+  const prompt = [
+    `Write OpenSCAD source for ${desc}.`,
+    "Requirements:",
+    `- footprint roughly ${w} (X) by ${d} (Y) units`,
+    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,
+    "- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)",
+    "- hollow shell (walls with an interior cavity), not a solid block",
+    "- a door opening and several windows",
+    "Output ONLY the .scad source, no prose, no code fences."
+  ].join("\n");
+  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } };
+}
+var BUILDING_CONFIGS = [
+  { id: "cottage", desc: "a single-story cottage", w: 50, d: 40, sh: 30, ns: 1, rh: 18 },
+  { id: "cabin", desc: "a small one-room cabin", w: 44, d: 36, sh: 26, ns: 1, rh: 22 },
+  { id: "bungalow", desc: "a wide single-story bungalow", w: 70, d: 52, sh: 28, ns: 1, rh: 20 },
+  { id: "barn", desc: "a long deep barn", w: 60, d: 96, sh: 42, ns: 1, rh: 26 },
+  { id: "warehouse", desc: "a large warehouse", w: 110, d: 80, sh: 40, ns: 1, rh: 26 },
+  { id: "chapel", desc: "a narrow tall chapel with a steep roof", w: 40, d: 72, sh: 48, ns: 1, rh: 34 },
+  { id: "farmhouse", desc: "a two-story farmhouse", w: 72, d: 56, sh: 30, ns: 2, rh: 26 },
+  { id: "two-story-manor", desc: "a wide two-story manor", w: 96, d: 64, sh: 32, ns: 2, rh: 28 },
+  { id: "townhouse", desc: "a narrow three-story townhouse", w: 42, d: 52, sh: 30, ns: 3, rh: 34 },
+  { id: "watchtower", desc: "a tall narrow three-story watchtower", w: 34, d: 34, sh: 30, ns: 3, rh: 32 }
+];
+var WATER_TOWER_GOLD = `
+$fn=48; r=24; bh=72; t=3; ch=26;
+union(){
+  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }
+  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);
+}
+`.trim();
+var AFRAME_GOLD = `
+w=44; d=60; h=66; t=3;
+module tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }
+  translate([w/2-7,-1,0]) cube([14,t+2,24]);
+}
+`.trim();
+var TASKS = [
+  {
+    id: "two-story-house",
+    prompt: [
+      "Write OpenSCAD source for a two-story house.",
+      "Requirements:",
+      "- footprint roughly 80 (X) by 60 (Y) units",
+      "- two stories, total wall height ~68 units",
+      "- a PITCHED / gabled roof on top (NOT a flat roof) \u2014 the roof must taper toward a ridge",
+      "- hollow shell (walls with an interior cavity), not a solid block",
+      "- at least one door opening and several windows",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 12],
+        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },
+        minTriangles: 60,
+        pitchedRoof: 0.55,
+        hollowBelow: 0.7
+      },
+      gold: `
+w=80; d=60; sh=34; t=3; rh=28;
+module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
+module roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
+union(){
+  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);
+  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);
+  translate([0,0,2*sh]) roof(w,d,rh);
+  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);
+  translate([w/2-8,-1,2]) cube([16,t+2,22]);
+}
+`.trim()
+    }
+  },
+  ...BUILDING_CONFIGS.map(buildingTask),
+  {
+    id: "water-tower",
+    prompt: [
+      "Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.",
+      "Requirements:",
+      "- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity",
+      "- a CONICAL roof on top that tapers to a point (a pitched roof)",
+      "- wall thickness around 3 units",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 4],
+        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },
+        minTriangles: 60,
+        pitchedRoof: 0.55,
+        hollowBelow: 0.8
+      },
+      gold: WATER_TOWER_GOLD
+    }
+  },
+  {
+    id: "a-frame-cabin",
+    prompt: [
+      "Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.",
+      "Requirements:",
+      "- footprint roughly 44 (X) by 60 (Y) units",
+      "- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top",
+      "- hollow inside (an interior cavity), not a solid wedge",
+      "- a door opening at the front",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 4],
+        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },
+        minTriangles: 30,
+        pitchedRoof: 0.45,
+        hollowBelow: 0.8
+      },
+      gold: AFRAME_GOLD
+    }
+  },
+  {
+    id: "hex-planter",
+    prompt: [
+      "Write OpenSCAD source for a hexagonal planter pot.",
+      "Requirements:",
+      "- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45",
+      "- open at the top, closed at the bottom (a cavity for soil)",
+      "- a wall thickness around 3 units",
+      "Output ONLY the .scad source, no prose, no code fences."
+    ].join("\n"),
+    meta: {
+      spec: {
+        volumes: [1, 3],
+        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },
+        minTriangles: 30,
+        hollowBelow: 0.7
+      },
+      gold: `
+$fn=6; or=25; h=45; t=3;
+difference(){
+  cylinder(h=h, r=or);
+  translate([0,0,t]) cylinder(h=h, r=or-t);
+}
+`.trim()
+    }
+  }
+];
+function createCadDesignAdapter() {
+  return {
+    name: "cad-design",
+    async preflight() {
+      try {
+        await execFileAsync("xvfb-run", ["-a", "openscad", "--version"], { timeout: 3e4 });
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `cad-design preflight failed: ${msg}
+Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). The judge runs \`xvfb-run -a openscad -o out.stl model.scad\` \u2014 both must be on PATH.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      let tasks = TASKS;
+      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
+      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
+      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: t.meta }));
+    },
+    async goldArtifact(task) {
+      const meta = task.metadata;
+      return meta?.gold;
+    },
+    async judge(task, artifact) {
+      const spec = task.metadata.spec;
+      const src = artifact.trim();
+      if (!src) return { resolved: false, score: 0, detail: "empty artifact" };
+      const dir = await mkdtemp(join(tmpdir(), "cad-"));
+      const scad = join(dir, "model.scad");
+      const stlPath = join(dir, "model.stl");
+      const pngPath = join(dir, "model.png");
+      await writeFile(scad, src);
+      let stderr = "";
+      try {
+        const r = await openscad(["-o", stlPath, scad], dir);
+        stderr = r.stderr;
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` };
+      }
+      try {
+        await openscad(
+          ["-o", pngPath, "--imgsize=1100,850", "--camera=40,30,40,55,0,25,260", "--colorscheme=Tomorrow", scad],
+          dir
+        );
+      } catch {
+      }
+      let stl;
+      try {
+        stl = await readFile(stlPath, "utf8");
+      } catch {
+        return { resolved: false, score: 0, detail: "compiled but produced no STL (empty geometry)" };
+      }
+      const volumes = Number(/Volumes:\s*(\d+)/.exec(stderr)?.[1] ?? "1");
+      const geo = measureStl(stl, volumes);
+      const { checks, score } = scoreGeometry(geo, spec);
+      const resolved = score === 1;
+      const detail = JSON.stringify({
+        checks,
+        geo: {
+          triangles: geo.triangles,
+          volumes: geo.volumes,
+          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },
+          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),
+          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null
+        },
+        renderPath: pngPath,
+        stlPath
+      });
+      return { resolved, score, detail };
+    }
+  };
+}
+
+export {
+  createCadDesignAdapter
+};
+//# sourceMappingURL=chunk-53UPUNBZ.js.map

package/dist/chunk-53UPUNBZ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/cad-design.ts"],"sourcesContent":["/**\n * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the\n * REAL OpenSCAD engine: compile + export STL, then measure the produced solid\n * geometry against the task's spec. Fully deterministic — the authoritative CAD\n * kernel is the gate, not an LLM and not the agent's own claim.\n *\n * This is verifiable-reward CAD: an agent is far better at *writing* exact\n * parametric code than at clicking a GUI, and the kernel either produces the\n * specified geometry or it doesn't. Every judged artifact also renders a PNG,\n * so a refine loop's attempts become a watchable \"model getting better\" reel\n * (run-capsule consumes the renders).\n *\n * Spec checks are geometric and ungameable:\n *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)\n *   - volumes       : disconnected-solid count within [min,max]\n *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)\n *   - detail        : triangle count ≥ floor (not a degenerate single cube)\n *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat\n *                     box can't fake this; a real gabled/hipped roof tapers\n *   - hollow        : interior cavity present (walls, not a solid block) — the\n *                     bbox volume materially exceeds the printed solid volume\n *\n * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Spec assertions a CAD task can require. All are deterministic + geometric. */\nexport interface CadSpec {\n  volumes?: [number, number]\n  bbox?: { x?: [number, number]; y?: [number, number]; z?: [number, number] }\n  minTriangles?: number\n  /** Top-band XY footprint must be < this fraction of the base footprint. */\n  pitchedRoof?: number\n  /** Printed solid volume must be < this fraction of the bbox volume (hollow). */\n  hollowBelow?: number\n}\n\ninterface CadTaskMeta {\n  spec: CadSpec\n  /** A known-good .scad that satisfies the spec — the oracle for verify-judge. */\n  gold: string\n}\n\n/** Run openscad with xvfb (it needs a GL context even headless). */\nasync function openscad(args: string[], cwd: string): Promise<{ stdout: string; stderr: string }> {\n  // -a => auto-pick a free display; openscad writes Facets/Volumes to stderr.\n  const { stdout, stderr } = await execFileAsync('xvfb-run', ['-a', 'openscad', ...args], {\n    cwd,\n    maxBuffer: 1024 * 1024 * 64,\n    timeout: 120_000,\n  })\n  return { stdout, stderr }\n}\n\ninterface Geometry {\n  triangles: number\n  volumes: number\n  bbox: { x: number; y: number; z: number }\n  /** XY footprint area (extent_x * extent_y) of the bottom 60% and top 25% z-bands. */\n  baseFootprint: number\n  topFootprint: number\n  /** Convex-ish solid volume estimate from the STL (sum of signed tetra volumes). */\n  solidVolume: number\n  bboxVolume: number\n}\n\n/** Parse an ASCII STL into the geometric measures the spec checks. Pure. */\nfunction measureStl(stl: string, volumesFromStderr: number): Geometry {\n  const verts: Array<[number, number, number]> = []\n  const tris: Array<[[number, number, number], [number, number, number], [number, number, number]]> = []\n  const re = /vertex\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)/g\n  let m: RegExpExecArray | null\n  const flat: Array<[number, number, number]> = []\n  while ((m = re.exec(stl))) flat.push([Number(m[1]), Number(m[2]), Number(m[3])])\n  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i]!, flat[i + 1]!, flat[i + 2]!])\n  for (const v of flat) verts.push(v)\n\n  const xs = verts.map((v) => v[0])\n  const ys = verts.map((v) => v[1])\n  const zs = verts.map((v) => v[2])\n  const min = (a: number[]) => Math.min(...a)\n  const max = (a: number[]) => Math.max(...a)\n  const minZ = min(zs)\n  const maxZ = max(zs)\n  const h = maxZ - minZ || 1\n\n  // Footprint of a z-band = XY extent of the vertices within it.\n  const footprint = (lo: number, hi: number): number => {\n    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h)\n    if (band.length < 3) return 0\n    const bx = band.map((v) => v[0])\n    const by = band.map((v) => v[1])\n    return (max(bx) - min(bx)) * (max(by) - min(by))\n  }\n\n  // Signed volume via the divergence theorem over triangles (|Σ v0·(v1×v2)/6|).\n  let vol6 = 0\n  for (const [a, b, c] of tris) {\n    vol6 +=\n      a[0] * (b[1] * c[2] - b[2] * c[1]) -\n      a[1] * (b[0] * c[2] - b[2] * c[0]) +\n      a[2] * (b[0] * c[1] - b[1] * c[0])\n  }\n  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ }\n  return {\n    triangles: tris.length,\n    volumes: volumesFromStderr,\n    bbox,\n    baseFootprint: footprint(0, 0.6),\n    topFootprint: footprint(0.75, 1),\n    solidVolume: Math.abs(vol6) / 6,\n    bboxVolume: bbox.x * bbox.y * bbox.z || 1,\n  }\n}\n\nfunction inRange(v: number, [lo, hi]: [number, number]): boolean {\n  return v >= lo && v <= hi\n}\n\n/** Score the geometry against the spec — each check is a named pass/fail. */\nfunction scoreGeometry(g: Geometry, spec: CadSpec): { checks: Record<string, boolean>; score: number } {\n  const checks: Record<string, boolean> = {}\n  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes)\n  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles\n  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x)\n  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y)\n  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z)\n  if (spec.pitchedRoof != null)\n    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint\n  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume\n  const vals = Object.values(checks)\n  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0\n  return { checks, score }\n}\n\n/** A parametric multi-story gabled building — the gold generator for the\n *  building-family tasks. They share geometry (hollow stacked shells + a gabled\n *  roof that tapers to a ridge + cut openings) and differ only in dimensions, so\n *  one verified generator yields a spec-passing oracle for each. */\nfunction buildingGold(w: number, d: number, sh: number, ns: number, rh: number, t = 3): string {\n  return `\nw=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);\n  translate([0,0,ns*sh]) roof(w,d,rh);\n  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);\n  translate([w/2-7,-1,2]) cube([14,t+2,20]);\n}\n`.trim()\n}\n\n/** Build a building-family task (prompt + spec + gold) from its dimensions.\n *  The spec's bbox is derived from the dims with a ±~18% tolerance so the agent\n *  must respect the brief's size; the gold (buildingGold) lands exactly on the\n *  nominal dims, so it passes by construction. Keep rh > ns*sh/3 (else the roof\n *  base sits inside the top-25% band and pitchedRoof fails). */\nfunction buildingTask(o: { id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }): {\n  id: string\n  prompt: string\n  meta: CadTaskMeta\n} {\n  const { id, desc, w, d, sh, ns, rh } = o\n  const totalWall = ns * sh\n  const totalH = totalWall + rh\n  const r = Math.round\n  const spec: CadSpec = {\n    volumes: [1, ns * 4 + 6],\n    bbox: {\n      x: [r(w * 0.82), r(w * 1.18)],\n      y: [r(d * 0.82), r((d + 1) * 1.2)],\n      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)],\n    },\n    minTriangles: 50,\n    pitchedRoof: 0.55,\n    hollowBelow: 0.72,\n  }\n  const prompt = [\n    `Write OpenSCAD source for ${desc}.`,\n    'Requirements:',\n    `- footprint roughly ${w} (X) by ${d} (Y) units`,\n    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,\n    '- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)',\n    '- hollow shell (walls with an interior cavity), not a solid block',\n    '- a door opening and several windows',\n    'Output ONLY the .scad source, no prose, no code fences.',\n  ].join('\\n')\n  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } }\n}\n\n/** The building family — one parametric skill (hit the bbox + pitched roof +\n *  hollow shell), sampled across footprints, heights, story counts, roof rises. */\nconst BUILDING_CONFIGS: Array<{ id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }> = [\n  { id: 'cottage', desc: 'a single-story cottage', w: 50, d: 40, sh: 30, ns: 1, rh: 18 },\n  { id: 'cabin', desc: 'a small one-room cabin', w: 44, d: 36, sh: 26, ns: 1, rh: 22 },\n  { id: 'bungalow', desc: 'a wide single-story bungalow', w: 70, d: 52, sh: 28, ns: 1, rh: 20 },\n  { id: 'barn', desc: 'a long deep barn', w: 60, d: 96, sh: 42, ns: 1, rh: 26 },\n  { id: 'warehouse', desc: 'a large warehouse', w: 110, d: 80, sh: 40, ns: 1, rh: 26 },\n  { id: 'chapel', desc: 'a narrow tall chapel with a steep roof', w: 40, d: 72, sh: 48, ns: 1, rh: 34 },\n  { id: 'farmhouse', desc: 'a two-story farmhouse', w: 72, d: 56, sh: 30, ns: 2, rh: 26 },\n  { id: 'two-story-manor', desc: 'a wide two-story manor', w: 96, d: 64, sh: 32, ns: 2, rh: 28 },\n  { id: 'townhouse', desc: 'a narrow three-story townhouse', w: 42, d: 52, sh: 30, ns: 3, rh: 34 },\n  { id: 'watchtower', desc: 'a tall narrow three-story watchtower', w: 34, d: 34, sh: 30, ns: 3, rh: 32 },\n]\n\n/** A hollow cylindrical water tower with a conical roof — a non-box shape that\n *  still exercises pitched-roof (the cone tapers to an apex) + hollow + bbox. */\nconst WATER_TOWER_GOLD = `\n$fn=48; r=24; bh=72; t=3; ch=26;\nunion(){\n  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }\n  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);\n}\n`.trim()\n\n/** An A-frame: the steep roof IS the walls — a hollow triangular prism. The most\n *  extreme taper in the set; tests that the agent can build a non-rectangular shell. */\nconst AFRAME_GOLD = `\nw=44; d=60; h=66; t=3;\nmodule tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }\n  translate([w/2-7,-1,0]) cube([14,t+2,24]);\n}\n`.trim()\n\nconst TASKS: Array<{ id: string; prompt: string; meta: CadTaskMeta }> = [\n  {\n    id: 'two-story-house',\n    prompt: [\n      'Write OpenSCAD source for a two-story house.',\n      'Requirements:',\n      '- footprint roughly 80 (X) by 60 (Y) units',\n      '- two stories, total wall height ~68 units',\n      '- a PITCHED / gabled roof on top (NOT a flat roof) — the roof must taper toward a ridge',\n      '- hollow shell (walls with an interior cavity), not a solid block',\n      '- at least one door opening and several windows',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 12],\n        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.7,\n      },\n      gold: `\nw=80; d=60; sh=34; t=3; rh=28;\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);\n  translate([0,0,2*sh]) roof(w,d,rh);\n  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);\n  translate([w/2-8,-1,2]) cube([16,t+2,22]);\n}\n`.trim(),\n    },\n  },\n  ...BUILDING_CONFIGS.map(buildingTask),\n  {\n    id: 'water-tower',\n    prompt: [\n      'Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.',\n      'Requirements:',\n      '- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity',\n      '- a CONICAL roof on top that tapers to a point (a pitched roof)',\n      '- wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.8,\n      },\n      gold: WATER_TOWER_GOLD,\n    },\n  },\n  {\n    id: 'a-frame-cabin',\n    prompt: [\n      'Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.',\n      'Requirements:',\n      '- footprint roughly 44 (X) by 60 (Y) units',\n      '- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top',\n      '- hollow inside (an interior cavity), not a solid wedge',\n      '- a door opening at the front',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },\n        minTriangles: 30,\n        pitchedRoof: 0.45,\n        hollowBelow: 0.8,\n      },\n      gold: AFRAME_GOLD,\n    },\n  },\n  {\n    id: 'hex-planter',\n    prompt: [\n      'Write OpenSCAD source for a hexagonal planter pot.',\n      'Requirements:',\n      '- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45',\n      '- open at the top, closed at the bottom (a cavity for soil)',\n      '- a wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 3],\n        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },\n        minTriangles: 30,\n        hollowBelow: 0.7,\n      },\n      gold: `\n$fn=6; or=25; h=45; t=3;\ndifference(){\n  cylinder(h=h, r=or);\n  translate([0,0,t]) cylinder(h=h, r=or-t);\n}\n`.trim(),\n    },\n  },\n]\n\nexport function createCadDesignAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cad-design',\n\n    async preflight() {\n      try {\n        await execFileAsync('xvfb-run', ['-a', 'openscad', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `cad-design preflight failed: ${msg}\\n` +\n            `Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). ` +\n            `The judge runs \\`xvfb-run -a openscad -o out.stl model.scad\\` — both must be on PATH.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let tasks = TASKS\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: t.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = task.metadata as unknown as CadTaskMeta | undefined\n      return meta?.gold\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const spec = (task.metadata as unknown as CadTaskMeta).spec\n      const src = artifact.trim()\n      if (!src) return { resolved: false, score: 0, detail: 'empty artifact' }\n\n      const dir = await mkdtemp(join(tmpdir(), 'cad-'))\n      const scad = join(dir, 'model.scad')\n      const stlPath = join(dir, 'model.stl')\n      const pngPath = join(dir, 'model.png')\n      await writeFile(scad, src)\n\n      // GATE 1: compile + export STL. Nonzero exit / CGAL error => does not compile.\n      let stderr = ''\n      try {\n        const r = await openscad(['-o', stlPath, scad], dir)\n        stderr = r.stderr\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` }\n      }\n\n      // Render a PNG alongside (the artifact run-capsule animates). Non-fatal.\n      try {\n        await openscad(\n          ['-o', pngPath, '--imgsize=1100,850', '--camera=40,30,40,55,0,25,260', '--colorscheme=Tomorrow', scad],\n          dir,\n        )\n      } catch {\n        /* render is for the reel; geometry gate already has the STL */\n      }\n\n      let stl: string\n      try {\n        stl = await readFile(stlPath, 'utf8')\n      } catch {\n        return { resolved: false, score: 0, detail: 'compiled but produced no STL (empty geometry)' }\n      }\n      const volumes = Number(/Volumes:\\s*(\\d+)/.exec(stderr)?.[1] ?? '1')\n      const geo = measureStl(stl, volumes)\n      const { checks, score } = scoreGeometry(geo, spec)\n      const resolved = score === 1\n      const detail = JSON.stringify({\n        checks,\n        geo: {\n          triangles: geo.triangles,\n          volumes: geo.volumes,\n          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },\n          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),\n          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null,\n        },\n        renderPath: pngPath,\n        stlPath,\n      })\n      return { resolved, score, detail }\n    },\n  }\n}\n"],"mappings":";AAyBA,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,iBAAiB;AAC7C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AAoBxC,eAAe,SAAS,MAAgB,KAA0D;AAEhG,QAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,YAAY,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG;AAAA,IACtF;AAAA,IACA,WAAW,OAAO,OAAO;AAAA,IACzB,SAAS;AAAA,EACX,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO;AAC1B;AAeA,SAAS,WAAW,KAAa,mBAAqC;AACpE,QAAM,QAAyC,CAAC;AAChD,QAAM,OAA8F,CAAC;AACrG,QAAM,KAAK;AACX,MAAI;AACJ,QAAM,OAAwC,CAAC;AAC/C,SAAQ,IAAI,GAAG,KAAK,GAAG,EAAI,MAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/E,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK,QAAQ,KAAK,EAAG,MAAK,KAAK,CAAC,KAAK,CAAC,GAAI,KAAK,IAAI,CAAC,GAAI,KAAK,IAAI,CAAC,CAAE,CAAC;AAC7F,aAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAElC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,IAAI,OAAO,QAAQ;AAGzB,QAAM,YAAY,CAAC,IAAY,OAAuB;AACpD,UAAM,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,KAAK,EAAE,CAAC,KAAK,OAAO,KAAK,CAAC;AAC/E,QAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,YAAQ,IAAI,EAAE,IAAI,IAAI,EAAE,MAAM,IAAI,EAAE,IAAI,IAAI,EAAE;AAAA,EAChD;AAGA,MAAI,OAAO;AACX,aAAW,CAAC,GAAG,GAAG,CAAC,KAAK,MAAM;AAC5B,YACE,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpC;AACA,QAAM,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,OAAO,KAAK;AAC1E,SAAO;AAAA,IACL,WAAW,KAAK;AAAA,IAChB,SAAS;AAAA,IACT;AAAA,IACA,eAAe,UAAU,GAAG,GAAG;AAAA,IAC/B,cAAc,UAAU,MAAM,CAAC;AAAA,IAC/B,aAAa,KAAK,IAAI,IAAI,IAAI;AAAA,IAC9B,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,KAAK;AAAA,EAC1C;AACF;AAEA,SAAS,QAAQ,GAAW,CAAC,IAAI,EAAE,GAA8B;AAC/D,SAAO,KAAK,MAAM,KAAK;AACzB;AAGA,SAAS,cAAc,GAAa,MAAmE;AACrG,QAAM,SAAkC,CAAC;AACzC,MAAI,KAAK,QAAS,QAAO,UAAU,QAAQ,EAAE,SAAS,KAAK,OAAO;AAClE,MAAI,KAAK,gBAAgB,KAAM,QAAO,SAAS,EAAE,aAAa,KAAK;AACnE,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,eAAe;AACtB,WAAO,cAAc,EAAE,gBAAgB,KAAK,EAAE,eAAe,KAAK,cAAc,EAAE;AACpF,MAAI,KAAK,eAAe,KAAM,QAAO,SAAS,EAAE,cAAc,KAAK,cAAc,EAAE;AACnF,QAAM,OAAO,OAAO,OAAO,MAAM;AACjC,QAAM,QAAQ,KAAK,SAAS,KAAK,OAAO,OAAO,EAAE,SAAS,KAAK,SAAS;AACxE,SAAO,EAAE,QAAQ,MAAM;AACzB;AAMA,SAAS,aAAa,GAAW,GAAW,IAAY,IAAY,IAAY,IAAI,GAAW;AAC7F,SAAO;AAAA,IACL,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUjD,KAAK;AACP;AAOA,SAAS,aAAa,GAIpB;AACA,QAAM,EAAE,IAAI,MAAM,GAAG,GAAG,IAAI,IAAI,GAAG,IAAI;AACvC,QAAM,YAAY,KAAK;AACvB,QAAM,SAAS,YAAY;AAC3B,QAAM,IAAI,KAAK;AACf,QAAM,OAAgB;AAAA,IACpB,SAAS,CAAC,GAAG,KAAK,IAAI,CAAC;AAAA,IACvB,MAAM;AAAA,MACJ,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,CAAC;AAAA,MAC5B,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC;AAAA,MACjC,GAAG,CAAC,EAAE,SAAS,IAAI,GAAG,GAAG,SAAS,KAAK,GAAG,CAAC;AAAA,IAC7C;AAAA,IACA,cAAc;AAAA,IACd,aAAa;AAAA,IACb,aAAa;AAAA,EACf;AACA,QAAM,SAAS;AAAA,IACb,6BAA6B,IAAI;AAAA,IACjC;AAAA,IACA,uBAAuB,CAAC,WAAW,CAAC;AAAA,IACpC,KAAK,IAAI,KAAK,EAAE,gCAAgC,SAAS,WAAW,6BAA6B,EAAE;AAAA,IACnG;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACX,SAAO,EAAE,IAAI,QAAQ,MAAM,EAAE,MAAM,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE;AAC5E;AAIA,IAAM,mBAAkH;AAAA,EACtH,EAAE,IAAI,WAAW,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACrF,EAAE,IAAI,SAAS,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,YAAY,MAAM,gCAAgC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5F,EAAE,IAAI,QAAQ,MAAM,oBAAoB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5E,EAAE,IAAI,aAAa,MAAM,qBAAqB,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,UAAU,MAAM,0CAA0C,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACpG,EAAE,IAAI,aAAa,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACtF,EAAE,IAAI,mBAAmB,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC7F,EAAE,IAAI,aAAa,MAAM,kCAAkC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC/F,EAAE,IAAI,cAAc,MAAM,wCAAwC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AACxG;AAIA,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMvB,KAAK;AAIP,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOlB,KAAK;AAEP,IAAM,QAAkE;AAAA,EACtE;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,EAAE;AAAA,QACf,MAAM,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAChD,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWV,KAAK;AAAA,IACH;AAAA,EACF;AAAA,EACA,GAAG,iBAAiB,IAAI,YAAY;AAAA,EACpC;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAC/C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMV,KAAK;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM,cAAc,YAAY,CAAC,MAAM,YAAY,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MACtF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,gCAAgC,GAAG;AAAA;AAAA,QAGrC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,QAAQ;AACZ,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,EAAE,KAA2C,EAAE;AAAA,IAC7H;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK;AAClB,aAAO,MAAM;AAAA,IACf;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAQ,KAAK,SAAoC;AACvD,YAAM,MAAM,SAAS,KAAK;AAC1B,UAAI,CAAC,IAAK,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AAEvE,YAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,MAAM,CAAC;AAChD,YAAM,OAAO,KAAK,KAAK,YAAY;AACnC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,MAAM,GAAG;AAGzB,UAAI,SAAS;AACb,UAAI;AACF,cAAM,IAAI,MAAM,SAAS,CAAC,MAAM,SAAS,IAAI,GAAG,GAAG;AACnD,iBAAS,EAAE;AAAA,MACb,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,MACrF;AAGA,UAAI;AACF,cAAM;AAAA,UACJ,CAAC,MAAM,SAAS,sBAAsB,iCAAiC,0BAA0B,IAAI;AAAA,UACrG;AAAA,QACF;AAAA,MACF,QAAQ;AAAA,MAER;AAEA,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,SAAS,SAAS,MAAM;AAAA,MACtC,QAAQ;AACN,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gDAAgD;AAAA,MAC9F;AACA,YAAM,UAAU,OAAO,mBAAmB,KAAK,MAAM,IAAI,CAAC,KAAK,GAAG;AAClE,YAAM,MAAM,WAAW,KAAK,OAAO;AACnC,YAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,KAAK,IAAI;AACjD,YAAM,WAAW,UAAU;AAC3B,YAAM,SAAS,KAAK,UAAU;AAAA,QAC5B;AAAA,QACA,KAAK;AAAA,UACH,WAAW,IAAI;AAAA,UACf,SAAS,IAAI;AAAA,UACb,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,EAAE;AAAA,UACxF,aAAa,EAAE,IAAI,cAAc,IAAI,YAAY,QAAQ,CAAC;AAAA,UAC1D,oBAAoB,IAAI,gBAAgB,EAAE,IAAI,eAAe,IAAI,eAAe,QAAQ,CAAC,IAAI;AAAA,QAC/F;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,MACF,CAAC;AACD,aAAO,EAAE,UAAU,OAAO,OAAO;AAAA,IACnC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-5H5XV76F.js

@@ -0,0 +1,240 @@
+// src/benchmarks/humaneval.ts
+import { execFile } from "child_process";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { gunzipSync } from "zlib";
+var humanevalUrl = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz";
+var dockerImage = "python:3.12-slim";
+var dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 2e4);
+async function loadHumanEval(limit, offset = 0) {
+  const localGz = process.env.HUMANEVAL_GZ;
+  let gz;
+  if (localGz) {
+    gz = readFileSync(localGz);
+  } else {
+    const res = await fetch(humanevalUrl);
+    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`);
+    gz = Buffer.from(await res.arrayBuffer());
+  }
+  const text = gunzipSync(gz).toString("utf8");
+  const tasks = [];
+  for (const line of text.split("\n")) {
+    if (line.trim() === "") continue;
+    const d = JSON.parse(line);
+    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {
+      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`);
+    }
+    tasks.push({
+      taskId: d.task_id,
+      prompt: d.prompt,
+      test: d.test,
+      entryPoint: d.entry_point,
+      ...d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}
+    });
+  }
+  if (tasks.length === 0) throw new Error("HumanEval parsed to 0 tasks");
+  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`);
+  return tasks.slice(offset, offset + limit);
+}
+var solveInstruction = "Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.";
+function basePrompt(task) {
+  return `${solveInstruction}
+
+\`\`\`python
+${task.prompt}\`\`\``;
+}
+function extractCode(reply) {
+  const fenced = reply.match(/```(?:python|py)?\s*\n([\s\S]*?)```/i);
+  if (fenced && typeof fenced[1] === "string") return fenced[1].trim();
+  return reply.trim();
+}
+function buildProgram(task, candidate) {
+  return `${task.prompt}
+${candidate}
+
+${task.test}
+
+check(${task.entryPoint})
+`;
+}
+var dockerRunSeq = 0;
+function runPythonProgram(program, timeoutMs = dockerTimeoutMs) {
+  const dir = mkdtempSync(join(tmpdir(), "hev-"));
+  chmodSync(dir, 493);
+  writeFileSync(join(dir, "p.py"), program, { mode: 420 });
+  const name = `hev-${process.pid}-${dockerRunSeq++}`;
+  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`;
+  return new Promise((resolvePromise, reject) => {
+    let settled = false;
+    const cleanup = () => {
+      rmSync(dir, { recursive: true, force: true });
+      execFile("docker", ["rm", "-f", name], () => {
+      });
+    };
+    const finish = (res) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(backstop);
+      cleanup();
+      resolvePromise(res);
+    };
+    const fail = (e) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(backstop);
+      cleanup();
+      reject(e);
+    };
+    const outerTimeoutMs = timeoutMs + 3e3;
+    const backstop = setTimeout(
+      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3e3}ms`)),
+      outerTimeoutMs + 3e3
+    );
+    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1e3));
+    execFile(
+      "docker",
+      [
+        "run",
+        "--rm",
+        "--pull=never",
+        "--name",
+        name,
+        "--network=none",
+        "--cpus=1",
+        "--memory=512m",
+        "--pids-limit=64",
+        "--cap-drop=ALL",
+        "--security-opt=no-new-privileges",
+        "--read-only",
+        "--tmpfs",
+        "/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777",
+        "--user",
+        "65534:65534",
+        "-v",
+        `${dir}:/w:ro`,
+        "-w",
+        "/w",
+        dockerImage,
+        "sh",
+        "-c",
+        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf "%s\\n" "$1"; exec timeout -s KILL "$2" python /w/p.py',
+        "agent-runtime-checker",
+        startMarker,
+        `${inContainerSeconds}s`
+      ],
+      { timeout: outerTimeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024 },
+      (err, stdout, stderr) => {
+        const markerLine = `${startMarker}
+`;
+        const candidateStarted = stdout?.startsWith(markerLine) === true;
+        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : stdout ?? "";
+        if (err) {
+          const e = err;
+          if (e.code === "ENOENT") {
+            fail(new Error("docker binary not found on PATH \u2014 cannot run the deployable checker"));
+            return;
+          }
+          if (e.killed) {
+            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`));
+            return;
+          }
+          if (!candidateStarted) {
+            if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(
+              stderr
+            )) {
+              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
+              return;
+            }
+            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
+              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
+              return;
+            }
+            fail(
+              new Error(
+                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`
+              )
+            );
+            return;
+          }
+          const exitCode = typeof e.code === "number" ? e.code : 1;
+          finish({
+            exitCode,
+            stdout: candidateStdout,
+            stderr: stderr ?? ""
+          });
+          return;
+        }
+        if (!candidateStarted) {
+          fail(new Error("docker checker exited without starting the candidate"));
+          return;
+        }
+        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? "" });
+      }
+    );
+  });
+}
+async function runChecker(task, candidate) {
+  const result = await runPythonProgram(buildProgram(task, candidate));
+  return result.exitCode === 0 ? { pass: 1 } : { pass: 0, detail: result.stderr.slice(-600) || "timed out (no output)" };
+}
+function toBenchTask(t) {
+  const metadata = {
+    promptHeader: t.prompt,
+    test: t.test,
+    entryPoint: t.entryPoint,
+    ...t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}
+  };
+  return { id: t.taskId, prompt: basePrompt(t), metadata };
+}
+function taskFromMeta(task) {
+  const m = task.metadata;
+  if (!m?.promptHeader || !m.test || !m.entryPoint) {
+    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`);
+  }
+  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint };
+}
+function createHumanEvalAdapter() {
+  return {
+    name: "humaneval",
+    async preflight() {
+      await new Promise((resolve, reject) => {
+        execFile("docker", ["version", "--format", "{{.Server.Version}}"], (err) => {
+          if (err) reject(new Error("HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)"));
+          else resolve();
+        });
+      });
+      await new Promise((resolve, reject) => {
+        execFile("docker", ["image", "inspect", dockerImage], (err) => {
+          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`));
+          else resolve();
+        });
+      });
+    },
+    async loadTasks(opts) {
+      const offset = Number(process.env.OFFSET ?? 0);
+      const all = await loadHumanEval(opts?.ids ? 164 : opts?.limit ?? 8, offset);
+      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all;
+      return picked.map(toBenchTask);
+    },
+    async judge(task, artifact) {
+      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact));
+      return { resolved: pass === 1, score: pass, detail: pass === 1 ? "tests passed" : "tests failed" };
+    },
+    async goldArtifact(task) {
+      const m = task.metadata;
+      const sol = m?.canonicalSolution;
+      return sol ? `${m.promptHeader}${sol}` : void 0;
+    }
+  };
+}
+
+export {
+  loadHumanEval,
+  basePrompt,
+  extractCode,
+  runPythonProgram,
+  runChecker,
+  createHumanEvalAdapter
+};
+//# sourceMappingURL=chunk-5H5XV76F.js.map