@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-53UPUNBZ.js

@@ -1,325 +0,0 @@
-// src/benchmarks/cad-design.ts
-import { execFile } from "child_process";
-import { mkdtemp, readFile, writeFile } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-async function openscad(args, cwd) {
-  const { stdout, stderr } = await execFileAsync("xvfb-run", ["-a", "openscad", ...args], {
-    cwd,
-    maxBuffer: 1024 * 1024 * 64,
-    timeout: 12e4
-  });
-  return { stdout, stderr };
-}
-function measureStl(stl, volumesFromStderr) {
-  const verts = [];
-  const tris = [];
-  const re = /vertex\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)\s+(-?[\d.eE+]+)/g;
-  let m;
-  const flat = [];
-  while (m = re.exec(stl)) flat.push([Number(m[1]), Number(m[2]), Number(m[3])]);
-  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i], flat[i + 1], flat[i + 2]]);
-  for (const v of flat) verts.push(v);
-  const xs = verts.map((v) => v[0]);
-  const ys = verts.map((v) => v[1]);
-  const zs = verts.map((v) => v[2]);
-  const min = (a) => Math.min(...a);
-  const max = (a) => Math.max(...a);
-  const minZ = min(zs);
-  const maxZ = max(zs);
-  const h = maxZ - minZ || 1;
-  const footprint = (lo, hi) => {
-    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h);
-    if (band.length < 3) return 0;
-    const bx = band.map((v) => v[0]);
-    const by = band.map((v) => v[1]);
-    return (max(bx) - min(bx)) * (max(by) - min(by));
-  };
-  let vol6 = 0;
-  for (const [a, b, c] of tris) {
-    vol6 += a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0]);
-  }
-  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ };
-  return {
-    triangles: tris.length,
-    volumes: volumesFromStderr,
-    bbox,
-    baseFootprint: footprint(0, 0.6),
-    topFootprint: footprint(0.75, 1),
-    solidVolume: Math.abs(vol6) / 6,
-    bboxVolume: bbox.x * bbox.y * bbox.z || 1
-  };
-}
-function inRange(v, [lo, hi]) {
-  return v >= lo && v <= hi;
-}
-function scoreGeometry(g, spec) {
-  const checks = {};
-  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes);
-  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles;
-  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x);
-  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y);
-  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z);
-  if (spec.pitchedRoof != null)
-    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint;
-  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume;
-  const vals = Object.values(checks);
-  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0;
-  return { checks, score };
-}
-function buildingGold(w, d, sh, ns, rh, t = 3) {
-  return `
-w=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};
-module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
-module roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);
-  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);
-  translate([0,0,ns*sh]) roof(w,d,rh);
-  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);
-  translate([w/2-7,-1,2]) cube([14,t+2,20]);
-}
-`.trim();
-}
-function buildingTask(o) {
-  const { id, desc, w, d, sh, ns, rh } = o;
-  const totalWall = ns * sh;
-  const totalH = totalWall + rh;
-  const r = Math.round;
-  const spec = {
-    volumes: [1, ns * 4 + 6],
-    bbox: {
-      x: [r(w * 0.82), r(w * 1.18)],
-      y: [r(d * 0.82), r((d + 1) * 1.2)],
-      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)]
-    },
-    minTriangles: 50,
-    pitchedRoof: 0.55,
-    hollowBelow: 0.72
-  };
-  const prompt = [
-    `Write OpenSCAD source for ${desc}.`,
-    "Requirements:",
-    `- footprint roughly ${w} (X) by ${d} (Y) units`,
-    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,
-    "- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)",
-    "- hollow shell (walls with an interior cavity), not a solid block",
-    "- a door opening and several windows",
-    "Output ONLY the .scad source, no prose, no code fences."
-  ].join("\n");
-  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } };
-}
-var BUILDING_CONFIGS = [
-  { id: "cottage", desc: "a single-story cottage", w: 50, d: 40, sh: 30, ns: 1, rh: 18 },
-  { id: "cabin", desc: "a small one-room cabin", w: 44, d: 36, sh: 26, ns: 1, rh: 22 },
-  { id: "bungalow", desc: "a wide single-story bungalow", w: 70, d: 52, sh: 28, ns: 1, rh: 20 },
-  { id: "barn", desc: "a long deep barn", w: 60, d: 96, sh: 42, ns: 1, rh: 26 },
-  { id: "warehouse", desc: "a large warehouse", w: 110, d: 80, sh: 40, ns: 1, rh: 26 },
-  { id: "chapel", desc: "a narrow tall chapel with a steep roof", w: 40, d: 72, sh: 48, ns: 1, rh: 34 },
-  { id: "farmhouse", desc: "a two-story farmhouse", w: 72, d: 56, sh: 30, ns: 2, rh: 26 },
-  { id: "two-story-manor", desc: "a wide two-story manor", w: 96, d: 64, sh: 32, ns: 2, rh: 28 },
-  { id: "townhouse", desc: "a narrow three-story townhouse", w: 42, d: 52, sh: 30, ns: 3, rh: 34 },
-  { id: "watchtower", desc: "a tall narrow three-story watchtower", w: 34, d: 34, sh: 30, ns: 3, rh: 32 }
-];
-var WATER_TOWER_GOLD = `
-$fn=48; r=24; bh=72; t=3; ch=26;
-union(){
-  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }
-  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);
-}
-`.trim();
-var AFRAME_GOLD = `
-w=44; d=60; h=66; t=3;
-module tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }
-  translate([w/2-7,-1,0]) cube([14,t+2,24]);
-}
-`.trim();
-var TASKS = [
-  {
-    id: "two-story-house",
-    prompt: [
-      "Write OpenSCAD source for a two-story house.",
-      "Requirements:",
-      "- footprint roughly 80 (X) by 60 (Y) units",
-      "- two stories, total wall height ~68 units",
-      "- a PITCHED / gabled roof on top (NOT a flat roof) \u2014 the roof must taper toward a ridge",
-      "- hollow shell (walls with an interior cavity), not a solid block",
-      "- at least one door opening and several windows",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 12],
-        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },
-        minTriangles: 60,
-        pitchedRoof: 0.55,
-        hollowBelow: 0.7
-      },
-      gold: `
-w=80; d=60; sh=34; t=3; rh=28;
-module shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }
-module roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }
-union(){
-  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);
-  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);
-  translate([0,0,2*sh]) roof(w,d,rh);
-  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);
-  translate([w/2-8,-1,2]) cube([16,t+2,22]);
-}
-`.trim()
-    }
-  },
-  ...BUILDING_CONFIGS.map(buildingTask),
-  {
-    id: "water-tower",
-    prompt: [
-      "Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.",
-      "Requirements:",
-      "- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity",
-      "- a CONICAL roof on top that tapers to a point (a pitched roof)",
-      "- wall thickness around 3 units",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 4],
-        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },
-        minTriangles: 60,
-        pitchedRoof: 0.55,
-        hollowBelow: 0.8
-      },
-      gold: WATER_TOWER_GOLD
-    }
-  },
-  {
-    id: "a-frame-cabin",
-    prompt: [
-      "Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.",
-      "Requirements:",
-      "- footprint roughly 44 (X) by 60 (Y) units",
-      "- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top",
-      "- hollow inside (an interior cavity), not a solid wedge",
-      "- a door opening at the front",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 4],
-        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },
-        minTriangles: 30,
-        pitchedRoof: 0.45,
-        hollowBelow: 0.8
-      },
-      gold: AFRAME_GOLD
-    }
-  },
-  {
-    id: "hex-planter",
-    prompt: [
-      "Write OpenSCAD source for a hexagonal planter pot.",
-      "Requirements:",
-      "- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45",
-      "- open at the top, closed at the bottom (a cavity for soil)",
-      "- a wall thickness around 3 units",
-      "Output ONLY the .scad source, no prose, no code fences."
-    ].join("\n"),
-    meta: {
-      spec: {
-        volumes: [1, 3],
-        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },
-        minTriangles: 30,
-        hollowBelow: 0.7
-      },
-      gold: `
-$fn=6; or=25; h=45; t=3;
-difference(){
-  cylinder(h=h, r=or);
-  translate([0,0,t]) cylinder(h=h, r=or-t);
-}
-`.trim()
-    }
-  }
-];
-function createCadDesignAdapter() {
-  return {
-    name: "cad-design",
-    async preflight() {
-      try {
-        await execFileAsync("xvfb-run", ["-a", "openscad", "--version"], { timeout: 3e4 });
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `cad-design preflight failed: ${msg}
-Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). The judge runs \`xvfb-run -a openscad -o out.stl model.scad\` \u2014 both must be on PATH.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      let tasks = TASKS;
-      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
-      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
-      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: t.meta }));
-    },
-    async goldArtifact(task) {
-      const meta = task.metadata;
-      return meta?.gold;
-    },
-    async judge(task, artifact) {
-      const spec = task.metadata.spec;
-      const src = artifact.trim();
-      if (!src) return { resolved: false, score: 0, detail: "empty artifact" };
-      const dir = await mkdtemp(join(tmpdir(), "cad-"));
-      const scad = join(dir, "model.scad");
-      const stlPath = join(dir, "model.stl");
-      const pngPath = join(dir, "model.png");
-      await writeFile(scad, src);
-      let stderr = "";
-      try {
-        const r = await openscad(["-o", stlPath, scad], dir);
-        stderr = r.stderr;
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` };
-      }
-      try {
-        await openscad(
-          ["-o", pngPath, "--imgsize=1100,850", "--camera=40,30,40,55,0,25,260", "--colorscheme=Tomorrow", scad],
-          dir
-        );
-      } catch {
-      }
-      let stl;
-      try {
-        stl = await readFile(stlPath, "utf8");
-      } catch {
-        return { resolved: false, score: 0, detail: "compiled but produced no STL (empty geometry)" };
-      }
-      const volumes = Number(/Volumes:\s*(\d+)/.exec(stderr)?.[1] ?? "1");
-      const geo = measureStl(stl, volumes);
-      const { checks, score } = scoreGeometry(geo, spec);
-      const resolved = score === 1;
-      const detail = JSON.stringify({
-        checks,
-        geo: {
-          triangles: geo.triangles,
-          volumes: geo.volumes,
-          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },
-          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),
-          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null
-        },
-        renderPath: pngPath,
-        stlPath
-      });
-      return { resolved, score, detail };
-    }
-  };
-}
-
-export {
-  createCadDesignAdapter
-};
-//# sourceMappingURL=chunk-53UPUNBZ.js.map

package/dist/chunk-53UPUNBZ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/cad-design.ts"],"sourcesContent":["/**\n * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the\n * REAL OpenSCAD engine: compile + export STL, then measure the produced solid\n * geometry against the task's spec. Fully deterministic — the authoritative CAD\n * kernel is the gate, not an LLM and not the agent's own claim.\n *\n * This is verifiable-reward CAD: an agent is far better at *writing* exact\n * parametric code than at clicking a GUI, and the kernel either produces the\n * specified geometry or it doesn't. Every judged artifact also renders a PNG,\n * so a refine loop's attempts become a watchable \"model getting better\" reel\n * (run-capsule consumes the renders).\n *\n * Spec checks are geometric and ungameable:\n *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)\n *   - volumes       : disconnected-solid count within [min,max]\n *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)\n *   - detail        : triangle count ≥ floor (not a degenerate single cube)\n *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat\n *                     box can't fake this; a real gabled/hipped roof tapers\n *   - hollow        : interior cavity present (walls, not a solid block) — the\n *                     bbox volume materially exceeds the printed solid volume\n *\n * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Spec assertions a CAD task can require. All are deterministic + geometric. */\nexport interface CadSpec {\n  volumes?: [number, number]\n  bbox?: { x?: [number, number]; y?: [number, number]; z?: [number, number] }\n  minTriangles?: number\n  /** Top-band XY footprint must be < this fraction of the base footprint. */\n  pitchedRoof?: number\n  /** Printed solid volume must be < this fraction of the bbox volume (hollow). */\n  hollowBelow?: number\n}\n\ninterface CadTaskMeta {\n  spec: CadSpec\n  /** A known-good .scad that satisfies the spec — the oracle for verify-judge. */\n  gold: string\n}\n\n/** Run openscad with xvfb (it needs a GL context even headless). */\nasync function openscad(args: string[], cwd: string): Promise<{ stdout: string; stderr: string }> {\n  // -a => auto-pick a free display; openscad writes Facets/Volumes to stderr.\n  const { stdout, stderr } = await execFileAsync('xvfb-run', ['-a', 'openscad', ...args], {\n    cwd,\n    maxBuffer: 1024 * 1024 * 64,\n    timeout: 120_000,\n  })\n  return { stdout, stderr }\n}\n\ninterface Geometry {\n  triangles: number\n  volumes: number\n  bbox: { x: number; y: number; z: number }\n  /** XY footprint area (extent_x * extent_y) of the bottom 60% and top 25% z-bands. */\n  baseFootprint: number\n  topFootprint: number\n  /** Convex-ish solid volume estimate from the STL (sum of signed tetra volumes). */\n  solidVolume: number\n  bboxVolume: number\n}\n\n/** Parse an ASCII STL into the geometric measures the spec checks. Pure. */\nfunction measureStl(stl: string, volumesFromStderr: number): Geometry {\n  const verts: Array<[number, number, number]> = []\n  const tris: Array<[[number, number, number], [number, number, number], [number, number, number]]> = []\n  const re = /vertex\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)/g\n  let m: RegExpExecArray | null\n  const flat: Array<[number, number, number]> = []\n  while ((m = re.exec(stl))) flat.push([Number(m[1]), Number(m[2]), Number(m[3])])\n  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i]!, flat[i + 1]!, flat[i + 2]!])\n  for (const v of flat) verts.push(v)\n\n  const xs = verts.map((v) => v[0])\n  const ys = verts.map((v) => v[1])\n  const zs = verts.map((v) => v[2])\n  const min = (a: number[]) => Math.min(...a)\n  const max = (a: number[]) => Math.max(...a)\n  const minZ = min(zs)\n  const maxZ = max(zs)\n  const h = maxZ - minZ || 1\n\n  // Footprint of a z-band = XY extent of the vertices within it.\n  const footprint = (lo: number, hi: number): number => {\n    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h)\n    if (band.length < 3) return 0\n    const bx = band.map((v) => v[0])\n    const by = band.map((v) => v[1])\n    return (max(bx) - min(bx)) * (max(by) - min(by))\n  }\n\n  // Signed volume via the divergence theorem over triangles (|Σ v0·(v1×v2)/6|).\n  let vol6 = 0\n  for (const [a, b, c] of tris) {\n    vol6 +=\n      a[0] * (b[1] * c[2] - b[2] * c[1]) -\n      a[1] * (b[0] * c[2] - b[2] * c[0]) +\n      a[2] * (b[0] * c[1] - b[1] * c[0])\n  }\n  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ }\n  return {\n    triangles: tris.length,\n    volumes: volumesFromStderr,\n    bbox,\n    baseFootprint: footprint(0, 0.6),\n    topFootprint: footprint(0.75, 1),\n    solidVolume: Math.abs(vol6) / 6,\n    bboxVolume: bbox.x * bbox.y * bbox.z || 1,\n  }\n}\n\nfunction inRange(v: number, [lo, hi]: [number, number]): boolean {\n  return v >= lo && v <= hi\n}\n\n/** Score the geometry against the spec — each check is a named pass/fail. */\nfunction scoreGeometry(g: Geometry, spec: CadSpec): { checks: Record<string, boolean>; score: number } {\n  const checks: Record<string, boolean> = {}\n  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes)\n  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles\n  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x)\n  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y)\n  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z)\n  if (spec.pitchedRoof != null)\n    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint\n  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume\n  const vals = Object.values(checks)\n  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0\n  return { checks, score }\n}\n\n/** A parametric multi-story gabled building — the gold generator for the\n *  building-family tasks. They share geometry (hollow stacked shells + a gabled\n *  roof that tapers to a ridge + cut openings) and differ only in dimensions, so\n *  one verified generator yields a spec-passing oracle for each. */\nfunction buildingGold(w: number, d: number, sh: number, ns: number, rh: number, t = 3): string {\n  return `\nw=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);\n  translate([0,0,ns*sh]) roof(w,d,rh);\n  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);\n  translate([w/2-7,-1,2]) cube([14,t+2,20]);\n}\n`.trim()\n}\n\n/** Build a building-family task (prompt + spec + gold) from its dimensions.\n *  The spec's bbox is derived from the dims with a ±~18% tolerance so the agent\n *  must respect the brief's size; the gold (buildingGold) lands exactly on the\n *  nominal dims, so it passes by construction. Keep rh > ns*sh/3 (else the roof\n *  base sits inside the top-25% band and pitchedRoof fails). */\nfunction buildingTask(o: { id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }): {\n  id: string\n  prompt: string\n  meta: CadTaskMeta\n} {\n  const { id, desc, w, d, sh, ns, rh } = o\n  const totalWall = ns * sh\n  const totalH = totalWall + rh\n  const r = Math.round\n  const spec: CadSpec = {\n    volumes: [1, ns * 4 + 6],\n    bbox: {\n      x: [r(w * 0.82), r(w * 1.18)],\n      y: [r(d * 0.82), r((d + 1) * 1.2)],\n      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)],\n    },\n    minTriangles: 50,\n    pitchedRoof: 0.55,\n    hollowBelow: 0.72,\n  }\n  const prompt = [\n    `Write OpenSCAD source for ${desc}.`,\n    'Requirements:',\n    `- footprint roughly ${w} (X) by ${d} (Y) units`,\n    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,\n    '- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)',\n    '- hollow shell (walls with an interior cavity), not a solid block',\n    '- a door opening and several windows',\n    'Output ONLY the .scad source, no prose, no code fences.',\n  ].join('\\n')\n  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } }\n}\n\n/** The building family — one parametric skill (hit the bbox + pitched roof +\n *  hollow shell), sampled across footprints, heights, story counts, roof rises. */\nconst BUILDING_CONFIGS: Array<{ id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }> = [\n  { id: 'cottage', desc: 'a single-story cottage', w: 50, d: 40, sh: 30, ns: 1, rh: 18 },\n  { id: 'cabin', desc: 'a small one-room cabin', w: 44, d: 36, sh: 26, ns: 1, rh: 22 },\n  { id: 'bungalow', desc: 'a wide single-story bungalow', w: 70, d: 52, sh: 28, ns: 1, rh: 20 },\n  { id: 'barn', desc: 'a long deep barn', w: 60, d: 96, sh: 42, ns: 1, rh: 26 },\n  { id: 'warehouse', desc: 'a large warehouse', w: 110, d: 80, sh: 40, ns: 1, rh: 26 },\n  { id: 'chapel', desc: 'a narrow tall chapel with a steep roof', w: 40, d: 72, sh: 48, ns: 1, rh: 34 },\n  { id: 'farmhouse', desc: 'a two-story farmhouse', w: 72, d: 56, sh: 30, ns: 2, rh: 26 },\n  { id: 'two-story-manor', desc: 'a wide two-story manor', w: 96, d: 64, sh: 32, ns: 2, rh: 28 },\n  { id: 'townhouse', desc: 'a narrow three-story townhouse', w: 42, d: 52, sh: 30, ns: 3, rh: 34 },\n  { id: 'watchtower', desc: 'a tall narrow three-story watchtower', w: 34, d: 34, sh: 30, ns: 3, rh: 32 },\n]\n\n/** A hollow cylindrical water tower with a conical roof — a non-box shape that\n *  still exercises pitched-roof (the cone tapers to an apex) + hollow + bbox. */\nconst WATER_TOWER_GOLD = `\n$fn=48; r=24; bh=72; t=3; ch=26;\nunion(){\n  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }\n  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);\n}\n`.trim()\n\n/** An A-frame: the steep roof IS the walls — a hollow triangular prism. The most\n *  extreme taper in the set; tests that the agent can build a non-rectangular shell. */\nconst AFRAME_GOLD = `\nw=44; d=60; h=66; t=3;\nmodule tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }\n  translate([w/2-7,-1,0]) cube([14,t+2,24]);\n}\n`.trim()\n\nconst TASKS: Array<{ id: string; prompt: string; meta: CadTaskMeta }> = [\n  {\n    id: 'two-story-house',\n    prompt: [\n      'Write OpenSCAD source for a two-story house.',\n      'Requirements:',\n      '- footprint roughly 80 (X) by 60 (Y) units',\n      '- two stories, total wall height ~68 units',\n      '- a PITCHED / gabled roof on top (NOT a flat roof) — the roof must taper toward a ridge',\n      '- hollow shell (walls with an interior cavity), not a solid block',\n      '- at least one door opening and several windows',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 12],\n        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.7,\n      },\n      gold: `\nw=80; d=60; sh=34; t=3; rh=28;\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);\n  translate([0,0,2*sh]) roof(w,d,rh);\n  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);\n  translate([w/2-8,-1,2]) cube([16,t+2,22]);\n}\n`.trim(),\n    },\n  },\n  ...BUILDING_CONFIGS.map(buildingTask),\n  {\n    id: 'water-tower',\n    prompt: [\n      'Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.',\n      'Requirements:',\n      '- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity',\n      '- a CONICAL roof on top that tapers to a point (a pitched roof)',\n      '- wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.8,\n      },\n      gold: WATER_TOWER_GOLD,\n    },\n  },\n  {\n    id: 'a-frame-cabin',\n    prompt: [\n      'Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.',\n      'Requirements:',\n      '- footprint roughly 44 (X) by 60 (Y) units',\n      '- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top',\n      '- hollow inside (an interior cavity), not a solid wedge',\n      '- a door opening at the front',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },\n        minTriangles: 30,\n        pitchedRoof: 0.45,\n        hollowBelow: 0.8,\n      },\n      gold: AFRAME_GOLD,\n    },\n  },\n  {\n    id: 'hex-planter',\n    prompt: [\n      'Write OpenSCAD source for a hexagonal planter pot.',\n      'Requirements:',\n      '- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45',\n      '- open at the top, closed at the bottom (a cavity for soil)',\n      '- a wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 3],\n        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },\n        minTriangles: 30,\n        hollowBelow: 0.7,\n      },\n      gold: `\n$fn=6; or=25; h=45; t=3;\ndifference(){\n  cylinder(h=h, r=or);\n  translate([0,0,t]) cylinder(h=h, r=or-t);\n}\n`.trim(),\n    },\n  },\n]\n\nexport function createCadDesignAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cad-design',\n\n    async preflight() {\n      try {\n        await execFileAsync('xvfb-run', ['-a', 'openscad', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `cad-design preflight failed: ${msg}\\n` +\n            `Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). ` +\n            `The judge runs \\`xvfb-run -a openscad -o out.stl model.scad\\` — both must be on PATH.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let tasks = TASKS\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: t.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = task.metadata as unknown as CadTaskMeta | undefined\n      return meta?.gold\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const spec = (task.metadata as unknown as CadTaskMeta).spec\n      const src = artifact.trim()\n      if (!src) return { resolved: false, score: 0, detail: 'empty artifact' }\n\n      const dir = await mkdtemp(join(tmpdir(), 'cad-'))\n      const scad = join(dir, 'model.scad')\n      const stlPath = join(dir, 'model.stl')\n      const pngPath = join(dir, 'model.png')\n      await writeFile(scad, src)\n\n      // GATE 1: compile + export STL. Nonzero exit / CGAL error => does not compile.\n      let stderr = ''\n      try {\n        const r = await openscad(['-o', stlPath, scad], dir)\n        stderr = r.stderr\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` }\n      }\n\n      // Render a PNG alongside (the artifact run-capsule animates). Non-fatal.\n      try {\n        await openscad(\n          ['-o', pngPath, '--imgsize=1100,850', '--camera=40,30,40,55,0,25,260', '--colorscheme=Tomorrow', scad],\n          dir,\n        )\n      } catch {\n        /* render is for the reel; geometry gate already has the STL */\n      }\n\n      let stl: string\n      try {\n        stl = await readFile(stlPath, 'utf8')\n      } catch {\n        return { resolved: false, score: 0, detail: 'compiled but produced no STL (empty geometry)' }\n      }\n      const volumes = Number(/Volumes:\\s*(\\d+)/.exec(stderr)?.[1] ?? '1')\n      const geo = measureStl(stl, volumes)\n      const { checks, score } = scoreGeometry(geo, spec)\n      const resolved = score === 1\n      const detail = JSON.stringify({\n        checks,\n        geo: {\n          triangles: geo.triangles,\n          volumes: geo.volumes,\n          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },\n          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),\n          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null,\n        },\n        renderPath: pngPath,\n        stlPath,\n      })\n      return { resolved, score, detail }\n    },\n  }\n}\n"],"mappings":";AAyBA,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,iBAAiB;AAC7C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AAoBxC,eAAe,SAAS,MAAgB,KAA0D;AAEhG,QAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,YAAY,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG;AAAA,IACtF;AAAA,IACA,WAAW,OAAO,OAAO;AAAA,IACzB,SAAS;AAAA,EACX,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO;AAC1B;AAeA,SAAS,WAAW,KAAa,mBAAqC;AACpE,QAAM,QAAyC,CAAC;AAChD,QAAM,OAA8F,CAAC;AACrG,QAAM,KAAK;AACX,MAAI;AACJ,QAAM,OAAwC,CAAC;AAC/C,SAAQ,IAAI,GAAG,KAAK,GAAG,EAAI,MAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/E,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK,QAAQ,KAAK,EAAG,MAAK,KAAK,CAAC,KAAK,CAAC,GAAI,KAAK,IAAI,CAAC,GAAI,KAAK,IAAI,CAAC,CAAE,CAAC;AAC7F,aAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAElC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,IAAI,OAAO,QAAQ;AAGzB,QAAM,YAAY,CAAC,IAAY,OAAuB;AACpD,UAAM,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,KAAK,EAAE,CAAC,KAAK,OAAO,KAAK,CAAC;AAC/E,QAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,YAAQ,IAAI,EAAE,IAAI,IAAI,EAAE,MAAM,IAAI,EAAE,IAAI,IAAI,EAAE;AAAA,EAChD;AAGA,MAAI,OAAO;AACX,aAAW,CAAC,GAAG,GAAG,CAAC,KAAK,MAAM;AAC5B,YACE,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpC;AACA,QAAM,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,OAAO,KAAK;AAC1E,SAAO;AAAA,IACL,WAAW,KAAK;AAAA,IAChB,SAAS;AAAA,IACT;AAAA,IACA,eAAe,UAAU,GAAG,GAAG;AAAA,IAC/B,cAAc,UAAU,MAAM,CAAC;AAAA,IAC/B,aAAa,KAAK,IAAI,IAAI,IAAI;AAAA,IAC9B,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,KAAK;AAAA,EAC1C;AACF;AAEA,SAAS,QAAQ,GAAW,CAAC,IAAI,EAAE,GAA8B;AAC/D,SAAO,KAAK,MAAM,KAAK;AACzB;AAGA,SAAS,cAAc,GAAa,MAAmE;AACrG,QAAM,SAAkC,CAAC;AACzC,MAAI,KAAK,QAAS,QAAO,UAAU,QAAQ,EAAE,SAAS,KAAK,OAAO;AAClE,MAAI,KAAK,gBAAgB,KAAM,QAAO,SAAS,EAAE,aAAa,KAAK;AACnE,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,eAAe;AACtB,WAAO,cAAc,EAAE,gBAAgB,KAAK,EAAE,eAAe,KAAK,cAAc,EAAE;AACpF,MAAI,KAAK,eAAe,KAAM,QAAO,SAAS,EAAE,cAAc,KAAK,cAAc,EAAE;AACnF,QAAM,OAAO,OAAO,OAAO,MAAM;AACjC,QAAM,QAAQ,KAAK,SAAS,KAAK,OAAO,OAAO,EAAE,SAAS,KAAK,SAAS;AACxE,SAAO,EAAE,QAAQ,MAAM;AACzB;AAMA,SAAS,aAAa,GAAW,GAAW,IAAY,IAAY,IAAY,IAAI,GAAW;AAC7F,SAAO;AAAA,IACL,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUjD,KAAK;AACP;AAOA,SAAS,aAAa,GAIpB;AACA,QAAM,EAAE,IAAI,MAAM,GAAG,GAAG,IAAI,IAAI,GAAG,IAAI;AACvC,QAAM,YAAY,KAAK;AACvB,QAAM,SAAS,YAAY;AAC3B,QAAM,IAAI,KAAK;AACf,QAAM,OAAgB;AAAA,IACpB,SAAS,CAAC,GAAG,KAAK,IAAI,CAAC;AAAA,IACvB,MAAM;AAAA,MACJ,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,CAAC;AAAA,MAC5B,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC;AAAA,MACjC,GAAG,CAAC,EAAE,SAAS,IAAI,GAAG,GAAG,SAAS,KAAK,GAAG,CAAC;AAAA,IAC7C;AAAA,IACA,cAAc;AAAA,IACd,aAAa;AAAA,IACb,aAAa;AAAA,EACf;AACA,QAAM,SAAS;AAAA,IACb,6BAA6B,IAAI;AAAA,IACjC;AAAA,IACA,uBAAuB,CAAC,WAAW,CAAC;AAAA,IACpC,KAAK,IAAI,KAAK,EAAE,gCAAgC,SAAS,WAAW,6BAA6B,EAAE;AAAA,IACnG;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACX,SAAO,EAAE,IAAI,QAAQ,MAAM,EAAE,MAAM,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE;AAC5E;AAIA,IAAM,mBAAkH;AAAA,EACtH,EAAE,IAAI,WAAW,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACrF,EAAE,IAAI,SAAS,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,YAAY,MAAM,gCAAgC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5F,EAAE,IAAI,QAAQ,MAAM,oBAAoB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5E,EAAE,IAAI,aAAa,MAAM,qBAAqB,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,UAAU,MAAM,0CAA0C,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACpG,EAAE,IAAI,aAAa,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACtF,EAAE,IAAI,mBAAmB,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC7F,EAAE,IAAI,aAAa,MAAM,kCAAkC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC/F,EAAE,IAAI,cAAc,MAAM,wCAAwC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AACxG;AAIA,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMvB,KAAK;AAIP,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOlB,KAAK;AAEP,IAAM,QAAkE;AAAA,EACtE;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,EAAE;AAAA,QACf,MAAM,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAChD,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWV,KAAK;AAAA,IACH;AAAA,EACF;AAAA,EACA,GAAG,iBAAiB,IAAI,YAAY;AAAA,EACpC;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAC/C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMV,KAAK;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM,cAAc,YAAY,CAAC,MAAM,YAAY,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MACtF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,gCAAgC,GAAG;AAAA;AAAA,QAGrC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,QAAQ;AACZ,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,EAAE,KAA2C,EAAE;AAAA,IAC7H;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK;AAClB,aAAO,MAAM;AAAA,IACf;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAQ,KAAK,SAAoC;AACvD,YAAM,MAAM,SAAS,KAAK;AAC1B,UAAI,CAAC,IAAK,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AAEvE,YAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,MAAM,CAAC;AAChD,YAAM,OAAO,KAAK,KAAK,YAAY;AACnC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,MAAM,GAAG;AAGzB,UAAI,SAAS;AACb,UAAI;AACF,cAAM,IAAI,MAAM,SAAS,CAAC,MAAM,SAAS,IAAI,GAAG,GAAG;AACnD,iBAAS,EAAE;AAAA,MACb,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,MACrF;AAGA,UAAI;AACF,cAAM;AAAA,UACJ,CAAC,MAAM,SAAS,sBAAsB,iCAAiC,0BAA0B,IAAI;AAAA,UACrG;AAAA,QACF;AAAA,MACF,QAAQ;AAAA,MAER;AAEA,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,SAAS,SAAS,MAAM;AAAA,MACtC,QAAQ;AACN,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gDAAgD;AAAA,MAC9F;AACA,YAAM,UAAU,OAAO,mBAAmB,KAAK,MAAM,IAAI,CAAC,KAAK,GAAG;AAClE,YAAM,MAAM,WAAW,KAAK,OAAO;AACnC,YAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,KAAK,IAAI;AACjD,YAAM,WAAW,UAAU;AAC3B,YAAM,SAAS,KAAK,UAAU;AAAA,QAC5B;AAAA,QACA,KAAK;AAAA,UACH,WAAW,IAAI;AAAA,UACf,SAAS,IAAI;AAAA,UACb,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,EAAE;AAAA,UACxF,aAAa,EAAE,IAAI,cAAc,IAAI,YAAY,QAAQ,CAAC;AAAA,UAC1D,oBAAoB,IAAI,gBAAgB,EAAE,IAAI,eAAe,IAAI,eAAe,QAAQ,CAAC,IAAI;AAAA,QAC/F;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,MACF,CAAC;AACD,aAAO,EAAE,UAAU,OAAO,OAAO;AAAA,IACnC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-5FEQDSCT.js

@@ -1,211 +0,0 @@
-import {
-  benchRoot,
-  runVenvScriptStdin
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/enterpriseops-gym.ts
-import { mkdir, readFile, rm, writeFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "enterpriseops-gym.json");
-var JUDGE = join(benchRoot, "scripts", "enterpriseops_gym_judge.py");
-var judgeCallSeq = 0;
-var DATASET = "ServiceNow-AI/EnterpriseOps-Gym";
-var DEFAULT_MODE = "oracle";
-var DEFAULT_DOMAIN = "itsm";
-var rowsApi = (mode, domain) => `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`;
-var enterpriseOpsTranscriptOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-function asArray(v) {
-  return typeof v === "string" ? JSON.parse(v) : v;
-}
-function workerContract(tools) {
-  return [
-    "",
-    `You have exactly these tools available (call NO others): ${tools.join(", ")}.`,
-    "Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.",
-    "Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:",
-    '{ "calls": [ { "tool": "<tool_name>", "arguments": { ... } } ] }',
-    "Include one entry per tool call in execution order. Nothing after the closing fence."
-  ].join("\n");
-}
-function rowToTask(row, mode) {
-  const servers = asArray(row.gym_servers_config);
-  const verifiers = asArray(row.verifiers);
-  const meta = {
-    taskId: row.task_id,
-    domain: row.domain,
-    mode,
-    selectedTools: row.selected_tools,
-    servers,
-    verifiers
-  };
-  return {
-    id: row.task_id,
-    split: row.domain,
-    prompt: [row.system_prompt, "", row.user_prompt, workerContract(row.selected_tools)].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "string" || !Array.isArray(md.servers) || !Array.isArray(md.verifiers) || md.verifiers.length === 0) {
-    throw new Error(`enterpriseops-gym task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, mode, opts) {
-  let tasks = rows.map((r) => rowToTask(r, mode));
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-async function loadFixtures(mode, opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(
-    `[enterpriseops-gym] EOPS_FIXTURES=1 \u2014 loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`
-  );
-  const domain = opts.split;
-  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows;
-  return selectRows(scoped, mode, opts);
-}
-async function fetchRows(mode, domain, opts) {
-  const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
-  const rows = [];
-  const want = opts.ids ? new Set(opts.ids) : null;
-  const page = 100;
-  const base = rowsApi(mode, domain);
-  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {
-    const res = await fetch(`${base}&offset=${offset}&length=${page}`);
-    if (!res.ok) {
-      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
-    }
-    const body = await res.json();
-    const got = body.rows ?? [];
-    if (got.length === 0) break;
-    for (const r of got) {
-      if (want && !want.has(r.row.task_id)) continue;
-      rows.push(r.row);
-    }
-    if (got.length < page) break;
-  }
-  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`);
-  return rows;
-}
-async function runJudge(meta, artifact) {
-  const taskJsonPath = join(
-    benchRoot,
-    ".eops-task-cache",
-    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`
-  );
-  await writeTaskCache(taskJsonPath, meta);
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(JUDGE, ["judge", "--task-json", taskJsonPath], artifact, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(
-      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}
-Fix: (1) run the ${meta.domain} gym server \u2014 docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; (2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir (the judge seeds each task's database from seed_database_file).`
-    );
-  } finally {
-    await rm(taskJsonPath, { force: true }).catch(() => {
-    });
-  }
-  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`);
-  if (typeof report.passes !== "number" || typeof report.total !== "number") {
-    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`);
-  }
-  const score = report.total > 0 ? report.passes / report.total : 0;
-  return {
-    resolved: report.success === true && report.total > 0 && report.passes === report.total,
-    score,
-    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total })
-  };
-}
-async function writeTaskCache(path, meta) {
-  await mkdir(join(path, ".."), { recursive: true });
-  await writeFile(
-    path,
-    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers })
-  );
-}
-async function probeServers(servers, domain) {
-  for (const s of servers) {
-    const url = `${s.mcp_server_url.replace(/\/$/, "")}/api/sql-runner`;
-    try {
-      const res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
-      void res.status;
-    } catch (err) {
-      throw new Error(
-        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}
-Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`
-      );
-    }
-  }
-}
-function createEnterpriseOpsGymAdapter() {
-  const fixturesMode = process.env.EOPS_FIXTURES === "1";
-  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE;
-  return {
-    name: "enterpriseops-gym",
-    output: enterpriseOpsTranscriptOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
-        });
-        return;
-      }
-      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => []);
-      const servers = sample[0] ? readMeta(sample[0]).servers : [];
-      if (servers.length === 0) {
-        throw new Error(
-          `enterpriseops-gym preflight: no gym_servers_config to probe. Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`
-        );
-      }
-      await probeServers(servers, DEFAULT_DOMAIN);
-    },
-    async loadTasks(opts = {}) {
-      const domain = opts.split ?? DEFAULT_DOMAIN;
-      if (fixturesMode) return loadFixtures(mode, opts);
-      let rows;
-      try {
-        rows = await fetchRows(mode, domain, opts);
-      } catch (err) {
-        console.warn(
-          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`
-        );
-        return loadFixtures(mode, opts);
-      }
-      return selectRows(rows, mode, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      return runJudge(meta, artifact);
-    }
-  };
-}
-
-export {
-  enterpriseOpsTranscriptOutput,
-  createEnterpriseOpsGymAdapter
-};
-//# sourceMappingURL=chunk-5FEQDSCT.js.map

package/dist/chunk-5FEQDSCT.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;AA8BA,SAAS,OAAO,UAAU,IAAI,iBAAiB;AAC/C,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;AAGrE,IAAI,eAAe;AAEnB,IAAM,UAAU;AAEhB,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAEvB,IAAM,UAAU,CAAC,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,CAAC,WAAW,mBAAmB,IAAI,CAAC,UAAU,mBAAmB,MAAM,CAAC;AA2CpJ,IAAM,gCAAuD;AAAA,EAClE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,QAAW,GAAsB;AACxC,SAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;AAC/C,SAAO;AAAA,IACL;AAAA,IACA,4DAA4D,MAAM,KAAK,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;AACxD,QAAM,UAAU,QAAyB,IAAI,kBAAkB;AAC/D,QAAM,YAAY,QAAkB,IAAI,SAAS;AACjD,QAAM,OAAiB;AAAA,IACrB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,eAAe,IAAI;AAAA,IACnB;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ,CAAC,IAAI,eAAe,IAAI,IAAI,aAAa,eAAe,IAAI,cAAc,CAAC,EAAE,KAAK,IAAI;AAAA,IAC9F,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GACvC;AACA,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;AACjF,MAAI,QAAQ,KAAK,IAAI,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAC9C,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;AACjF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,sDAAiD,KAAK,MAAM,+BAA+B,QAAQ;AAAA,EACrG;AACA,QAAM,SAAS,KAAK;AACpB,QAAM,SAAS,SAAS,KAAK,OAAO,CAAC,MAAM,EAAE,WAAW,MAAM,IAAI;AAClE,SAAO,WAAW,QAAQ,MAAM,IAAI;AACtC;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;AAC5F,QAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;AAC/D,QAAM,OAAkB,CAAC;AACzB,QAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;AAC5C,QAAM,OAAO;AACb,QAAM,OAAO,QAAQ,MAAM,MAAM;AACjC,WAAS,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;AAC1E,UAAM,MAAM,MAAM,MAAM,GAAG,IAAI,WAAW,MAAM,WAAW,IAAI,EAAE;AACjE,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,YAAY,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrH;AACA,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,MAAM,KAAK,QAAQ,CAAC;AAC1B,QAAI,IAAI,WAAW,EAAG;AACtB,eAAW,KAAK,KAAK;AACnB,UAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,EAAG;AACtC,WAAK,KAAK,EAAE,GAAG;AAAA,IACjB;AACA,QAAI,IAAI,SAAS,KAAM;AAAA,EACzB;AACA,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,QAAQ,IAAI,IAAI,MAAM,EAAE;AACzH,SAAO;AACT;AAYA,eAAe,SAAS,MAAgB,UAAuC;AAI7E,QAAM,eAAe;AAAA,IACnB;AAAA,IACA;AAAA,IACA,GAAG,KAAK,MAAM,IAAI,QAAQ,GAAG,IAAI,cAAc;AAAA,EACjD;AACA,QAAM,eAAe,cAAc,IAAI;AACvC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,OAAO,CAAC,SAAS,eAAe,YAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;AAAA,EAC/G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI;AAAA,MACR,sCAAsC,KAAK,MAAM,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC;AAAA,mBACzE,KAAK,MAAM,+EAC6B,KAAK,MAAM,uFACM,KAAK,MAAM;AAAA,IAG5F;AAAA,EACF,UAAE;AACA,UAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxD;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,qCAAqC,KAAK,MAAM,KAAK,OAAO,KAAK,EAAE;AACrG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAClF;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,KAAK,QAAQ,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;AACzE,QAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,QAAM;AAAA,IACJ;AAAA,IACA,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,QAAQ,KAAK,QAAQ,oBAAoB,KAAK,SAAS,WAAW,KAAK,UAAU,CAAC;AAAA,EAC3H;AACF;AAGA,eAAe,aAAa,SAA4B,QAA+B;AACrF,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,CAAC;AAClD,QAAI;AAEF,YAAM,MAAM,MAAM,MAAM,KAAK,EAAE,QAAQ,QAAQ,SAAS,EAAE,gBAAgB,mBAAmB,GAAG,MAAM,KAAK,CAAC;AAG5G,WAAK,IAAI;AAAA,IACX,SAAS,KAAK;AACZ,YAAM,IAAI;AAAA,QACR,gCAAgC,EAAE,eAAe,mBAAmB,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,gEACjD,MAAM,yFACQ,MAAM;AAAA,MAEzF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AACnD,QAAM,OAAO,QAAQ,IAAI,aAAa;AAEtC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAGhB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI,MAAM,uBAAuB,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,QAC3G,CAAC;AACD;AAAA,MACF;AAIA,YAAM,SAAS,MAAM,aAAa,MAAM,EAAE,OAAO,gBAAgB,OAAO,EAAE,CAAC,EAAE,MAAM,MAAM,CAAC,CAAC;AAC3F,YAAM,UAAU,OAAO,CAAC,IAAI,SAAS,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC;AAC3D,UAAI,QAAQ,WAAW,GAAG;AACxB,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,YAAM,aAAa,SAAS,cAAc;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,SAAS,KAAK,SAAS;AAC7B,UAAI,aAAc,QAAO,aAAa,MAAM,IAAI;AAChD,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAC3C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,GAAG,0CAA0C,QAAQ;AAAA,QAC3I;AACA,eAAO,aAAa,MAAM,IAAI;AAAA,MAChC;AACA,aAAO,WAAW,MAAM,MAAM,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,eAAe;AAMnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,SAAS,MAAM,QAAQ;AAAA,IAChC;AAAA,EACF;AACF;","names":[]}