@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-53UPUNBZ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/cad-design.ts"],"sourcesContent":["/**\n * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the\n * REAL OpenSCAD engine: compile + export STL, then measure the produced solid\n * geometry against the task's spec. Fully deterministic — the authoritative CAD\n * kernel is the gate, not an LLM and not the agent's own claim.\n *\n * This is verifiable-reward CAD: an agent is far better at *writing* exact\n * parametric code than at clicking a GUI, and the kernel either produces the\n * specified geometry or it doesn't. Every judged artifact also renders a PNG,\n * so a refine loop's attempts become a watchable \"model getting better\" reel\n * (run-capsule consumes the renders).\n *\n * Spec checks are geometric and ungameable:\n *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)\n *   - volumes       : disconnected-solid count within [min,max]\n *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)\n *   - detail        : triangle count ≥ floor (not a degenerate single cube)\n *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat\n *                     box can't fake this; a real gabled/hipped roof tapers\n *   - hollow        : interior cavity present (walls, not a solid block) — the\n *                     bbox volume materially exceeds the printed solid volume\n *\n * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Spec assertions a CAD task can require. All are deterministic + geometric. */\nexport interface CadSpec {\n  volumes?: [number, number]\n  bbox?: { x?: [number, number]; y?: [number, number]; z?: [number, number] }\n  minTriangles?: number\n  /** Top-band XY footprint must be < this fraction of the base footprint. */\n  pitchedRoof?: number\n  /** Printed solid volume must be < this fraction of the bbox volume (hollow). */\n  hollowBelow?: number\n}\n\ninterface CadTaskMeta {\n  spec: CadSpec\n  /** A known-good .scad that satisfies the spec — the oracle for verify-judge. */\n  gold: string\n}\n\n/** Run openscad with xvfb (it needs a GL context even headless). */\nasync function openscad(args: string[], cwd: string): Promise<{ stdout: string; stderr: string }> {\n  // -a => auto-pick a free display; openscad writes Facets/Volumes to stderr.\n  const { stdout, stderr } = await execFileAsync('xvfb-run', ['-a', 'openscad', ...args], {\n    cwd,\n    maxBuffer: 1024 * 1024 * 64,\n    timeout: 120_000,\n  })\n  return { stdout, stderr }\n}\n\ninterface Geometry {\n  triangles: number\n  volumes: number\n  bbox: { x: number; y: number; z: number }\n  /** XY footprint area (extent_x * extent_y) of the bottom 60% and top 25% z-bands. */\n  baseFootprint: number\n  topFootprint: number\n  /** Convex-ish solid volume estimate from the STL (sum of signed tetra volumes). */\n  solidVolume: number\n  bboxVolume: number\n}\n\n/** Parse an ASCII STL into the geometric measures the spec checks. Pure. */\nfunction measureStl(stl: string, volumesFromStderr: number): Geometry {\n  const verts: Array<[number, number, number]> = []\n  const tris: Array<[[number, number, number], [number, number, number], [number, number, number]]> = []\n  const re = /vertex\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)\\s+(-?[\\d.eE+]+)/g\n  let m: RegExpExecArray | null\n  const flat: Array<[number, number, number]> = []\n  while ((m = re.exec(stl))) flat.push([Number(m[1]), Number(m[2]), Number(m[3])])\n  for (let i = 0; i + 2 < flat.length; i += 3) tris.push([flat[i]!, flat[i + 1]!, flat[i + 2]!])\n  for (const v of flat) verts.push(v)\n\n  const xs = verts.map((v) => v[0])\n  const ys = verts.map((v) => v[1])\n  const zs = verts.map((v) => v[2])\n  const min = (a: number[]) => Math.min(...a)\n  const max = (a: number[]) => Math.max(...a)\n  const minZ = min(zs)\n  const maxZ = max(zs)\n  const h = maxZ - minZ || 1\n\n  // Footprint of a z-band = XY extent of the vertices within it.\n  const footprint = (lo: number, hi: number): number => {\n    const band = verts.filter((v) => v[2] >= minZ + lo * h && v[2] <= minZ + hi * h)\n    if (band.length < 3) return 0\n    const bx = band.map((v) => v[0])\n    const by = band.map((v) => v[1])\n    return (max(bx) - min(bx)) * (max(by) - min(by))\n  }\n\n  // Signed volume via the divergence theorem over triangles (|Σ v0·(v1×v2)/6|).\n  let vol6 = 0\n  for (const [a, b, c] of tris) {\n    vol6 +=\n      a[0] * (b[1] * c[2] - b[2] * c[1]) -\n      a[1] * (b[0] * c[2] - b[2] * c[0]) +\n      a[2] * (b[0] * c[1] - b[1] * c[0])\n  }\n  const bbox = { x: max(xs) - min(xs), y: max(ys) - min(ys), z: maxZ - minZ }\n  return {\n    triangles: tris.length,\n    volumes: volumesFromStderr,\n    bbox,\n    baseFootprint: footprint(0, 0.6),\n    topFootprint: footprint(0.75, 1),\n    solidVolume: Math.abs(vol6) / 6,\n    bboxVolume: bbox.x * bbox.y * bbox.z || 1,\n  }\n}\n\nfunction inRange(v: number, [lo, hi]: [number, number]): boolean {\n  return v >= lo && v <= hi\n}\n\n/** Score the geometry against the spec — each check is a named pass/fail. */\nfunction scoreGeometry(g: Geometry, spec: CadSpec): { checks: Record<string, boolean>; score: number } {\n  const checks: Record<string, boolean> = {}\n  if (spec.volumes) checks.volumes = inRange(g.volumes, spec.volumes)\n  if (spec.minTriangles != null) checks.detail = g.triangles >= spec.minTriangles\n  if (spec.bbox?.x) checks.bboxX = inRange(g.bbox.x, spec.bbox.x)\n  if (spec.bbox?.y) checks.bboxY = inRange(g.bbox.y, spec.bbox.y)\n  if (spec.bbox?.z) checks.bboxZ = inRange(g.bbox.z, spec.bbox.z)\n  if (spec.pitchedRoof != null)\n    checks.pitchedRoof = g.baseFootprint > 0 && g.topFootprint < spec.pitchedRoof * g.baseFootprint\n  if (spec.hollowBelow != null) checks.hollow = g.solidVolume < spec.hollowBelow * g.bboxVolume\n  const vals = Object.values(checks)\n  const score = vals.length ? vals.filter(Boolean).length / vals.length : 0\n  return { checks, score }\n}\n\n/** A parametric multi-story gabled building — the gold generator for the\n *  building-family tasks. They share geometry (hollow stacked shells + a gabled\n *  roof that tapers to a ridge + cut openings) and differ only in dimensions, so\n *  one verified generator yields a spec-passing oracle for each. */\nfunction buildingGold(w: number, d: number, sh: number, ns: number, rh: number, t = 3): string {\n  return `\nw=${w}; d=${d}; sh=${sh}; ns=${ns}; rh=${rh}; t=${t};\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0:ns-1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0:ns]) translate([0,0,s*sh-0.5]) cube([w,d,1]);\n  translate([0,0,ns*sh]) roof(w,d,rh);\n  for(s=[0:ns-1]) for(i=[0:floor((w-28)/22)]) translate([14+i*22,-1,s*sh+10]) cube([10,t+2,12]);\n  translate([w/2-7,-1,2]) cube([14,t+2,20]);\n}\n`.trim()\n}\n\n/** Build a building-family task (prompt + spec + gold) from its dimensions.\n *  The spec's bbox is derived from the dims with a ±~18% tolerance so the agent\n *  must respect the brief's size; the gold (buildingGold) lands exactly on the\n *  nominal dims, so it passes by construction. Keep rh > ns*sh/3 (else the roof\n *  base sits inside the top-25% band and pitchedRoof fails). */\nfunction buildingTask(o: { id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }): {\n  id: string\n  prompt: string\n  meta: CadTaskMeta\n} {\n  const { id, desc, w, d, sh, ns, rh } = o\n  const totalWall = ns * sh\n  const totalH = totalWall + rh\n  const r = Math.round\n  const spec: CadSpec = {\n    volumes: [1, ns * 4 + 6],\n    bbox: {\n      x: [r(w * 0.82), r(w * 1.18)],\n      y: [r(d * 0.82), r((d + 1) * 1.2)],\n      z: [r(totalH * 0.82), r((totalH + 1) * 1.2)],\n    },\n    minTriangles: 50,\n    pitchedRoof: 0.55,\n    hollowBelow: 0.72,\n  }\n  const prompt = [\n    `Write OpenSCAD source for ${desc}.`,\n    'Requirements:',\n    `- footprint roughly ${w} (X) by ${d} (Y) units`,\n    ns > 1 ? `- ${ns} stories, total wall height ~${totalWall} units` : `- one story, wall height ~${sh} units`,\n    '- a PITCHED / gabled roof on top that tapers toward a ridge (NOT flat)',\n    '- hollow shell (walls with an interior cavity), not a solid block',\n    '- a door opening and several windows',\n    'Output ONLY the .scad source, no prose, no code fences.',\n  ].join('\\n')\n  return { id, prompt, meta: { spec, gold: buildingGold(w, d, sh, ns, rh) } }\n}\n\n/** The building family — one parametric skill (hit the bbox + pitched roof +\n *  hollow shell), sampled across footprints, heights, story counts, roof rises. */\nconst BUILDING_CONFIGS: Array<{ id: string; desc: string; w: number; d: number; sh: number; ns: number; rh: number }> = [\n  { id: 'cottage', desc: 'a single-story cottage', w: 50, d: 40, sh: 30, ns: 1, rh: 18 },\n  { id: 'cabin', desc: 'a small one-room cabin', w: 44, d: 36, sh: 26, ns: 1, rh: 22 },\n  { id: 'bungalow', desc: 'a wide single-story bungalow', w: 70, d: 52, sh: 28, ns: 1, rh: 20 },\n  { id: 'barn', desc: 'a long deep barn', w: 60, d: 96, sh: 42, ns: 1, rh: 26 },\n  { id: 'warehouse', desc: 'a large warehouse', w: 110, d: 80, sh: 40, ns: 1, rh: 26 },\n  { id: 'chapel', desc: 'a narrow tall chapel with a steep roof', w: 40, d: 72, sh: 48, ns: 1, rh: 34 },\n  { id: 'farmhouse', desc: 'a two-story farmhouse', w: 72, d: 56, sh: 30, ns: 2, rh: 26 },\n  { id: 'two-story-manor', desc: 'a wide two-story manor', w: 96, d: 64, sh: 32, ns: 2, rh: 28 },\n  { id: 'townhouse', desc: 'a narrow three-story townhouse', w: 42, d: 52, sh: 30, ns: 3, rh: 34 },\n  { id: 'watchtower', desc: 'a tall narrow three-story watchtower', w: 34, d: 34, sh: 30, ns: 3, rh: 32 },\n]\n\n/** A hollow cylindrical water tower with a conical roof — a non-box shape that\n *  still exercises pitched-roof (the cone tapers to an apex) + hollow + bbox. */\nconst WATER_TOWER_GOLD = `\n$fn=48; r=24; bh=72; t=3; ch=26;\nunion(){\n  difference(){ cylinder(h=bh, r=r); translate([0,0,t]) cylinder(h=bh, r=r-t); }\n  translate([0,0,bh]) cylinder(h=ch, r1=r, r2=0);\n}\n`.trim()\n\n/** An A-frame: the steep roof IS the walls — a hollow triangular prism. The most\n *  extreme taper in the set; tests that the agent can build a non-rectangular shell. */\nconst AFRAME_GOLD = `\nw=44; d=60; h=66; t=3;\nmodule tri(w,d,h){ rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  difference(){ tri(w,d,h); translate([t,0,0]) tri(w-2*t,d,h-2*t*h/d); }\n  translate([w/2-7,-1,0]) cube([14,t+2,24]);\n}\n`.trim()\n\nconst TASKS: Array<{ id: string; prompt: string; meta: CadTaskMeta }> = [\n  {\n    id: 'two-story-house',\n    prompt: [\n      'Write OpenSCAD source for a two-story house.',\n      'Requirements:',\n      '- footprint roughly 80 (X) by 60 (Y) units',\n      '- two stories, total wall height ~68 units',\n      '- a PITCHED / gabled roof on top (NOT a flat roof) — the roof must taper toward a ridge',\n      '- hollow shell (walls with an interior cavity), not a solid block',\n      '- at least one door opening and several windows',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 12],\n        bbox: { x: [70, 110], y: [50, 80], z: [80, 130] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.7,\n      },\n      gold: `\nw=80; d=60; sh=34; t=3; rh=28;\nmodule shell(w,d,h,t){ difference(){ cube([w,d,h]); translate([t,t,-1]) cube([w-2*t,d-2*t,h+2]); } }\nmodule roof(w,d,h){ translate([0,0,0]) rotate([90,0,90]) linear_extrude(height=w) polygon([[0,0],[d,0],[d/2,h]]); }\nunion(){\n  for(s=[0,1]) translate([0,0,s*sh]) shell(w,d,sh,t);\n  for(s=[0,1,2]) translate([0,0,s*sh-1]) cube([w,d,1]);\n  translate([0,0,2*sh]) roof(w,d,rh);\n  for(s=[0,1]) for(i=[0:2]) translate([16+i*22,-1,s*sh+10]) cube([12,t+2,14]);\n  translate([w/2-8,-1,2]) cube([16,t+2,22]);\n}\n`.trim(),\n    },\n  },\n  ...BUILDING_CONFIGS.map(buildingTask),\n  {\n    id: 'water-tower',\n    prompt: [\n      'Write OpenSCAD source for a water tower: a tall hollow cylindrical tank with a conical roof.',\n      'Requirements:',\n      '- a hollow cylindrical tank, ~48 units in diameter, ~72 units tall, with an interior cavity',\n      '- a CONICAL roof on top that tapers to a point (a pitched roof)',\n      '- wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [40, 56], y: [40, 56], z: [88, 110] },\n        minTriangles: 60,\n        pitchedRoof: 0.55,\n        hollowBelow: 0.8,\n      },\n      gold: WATER_TOWER_GOLD,\n    },\n  },\n  {\n    id: 'a-frame-cabin',\n    prompt: [\n      'Write OpenSCAD source for an A-frame cabin: a steep triangular shell where the roof forms the walls.',\n      'Requirements:',\n      '- footprint roughly 44 (X) by 60 (Y) units',\n      '- a STEEP triangular cross-section ~66 units tall that tapers to a ridge at the top',\n      '- hollow inside (an interior cavity), not a solid wedge',\n      '- a door opening at the front',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 4],\n        bbox: { x: [36, 52], y: [50, 74], z: [56, 80] },\n        minTriangles: 30,\n        pitchedRoof: 0.45,\n        hollowBelow: 0.8,\n      },\n      gold: AFRAME_GOLD,\n    },\n  },\n  {\n    id: 'hex-planter',\n    prompt: [\n      'Write OpenSCAD source for a hexagonal planter pot.',\n      'Requirements:',\n      '- a hollow hexagonal prism (6-sided), outer width ~50 units across, height ~45',\n      '- open at the top, closed at the bottom (a cavity for soil)',\n      '- a wall thickness around 3 units',\n      'Output ONLY the .scad source, no prose, no code fences.',\n    ].join('\\n'),\n    meta: {\n      spec: {\n        volumes: [1, 3],\n        bbox: { x: [40, 60], y: [40, 60], z: [40, 55] },\n        minTriangles: 30,\n        hollowBelow: 0.7,\n      },\n      gold: `\n$fn=6; or=25; h=45; t=3;\ndifference(){\n  cylinder(h=h, r=or);\n  translate([0,0,t]) cylinder(h=h, r=or-t);\n}\n`.trim(),\n    },\n  },\n]\n\nexport function createCadDesignAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cad-design',\n\n    async preflight() {\n      try {\n        await execFileAsync('xvfb-run', ['-a', 'openscad', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `cad-design preflight failed: ${msg}\\n` +\n            `Fix: install OpenSCAD + Xvfb (Debian/Ubuntu: sudo apt-get install -y openscad xvfb). ` +\n            `The judge runs \\`xvfb-run -a openscad -o out.stl model.scad\\` — both must be on PATH.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let tasks = TASKS\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: t.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = task.metadata as unknown as CadTaskMeta | undefined\n      return meta?.gold\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const spec = (task.metadata as unknown as CadTaskMeta).spec\n      const src = artifact.trim()\n      if (!src) return { resolved: false, score: 0, detail: 'empty artifact' }\n\n      const dir = await mkdtemp(join(tmpdir(), 'cad-'))\n      const scad = join(dir, 'model.scad')\n      const stlPath = join(dir, 'model.stl')\n      const pngPath = join(dir, 'model.png')\n      await writeFile(scad, src)\n\n      // GATE 1: compile + export STL. Nonzero exit / CGAL error => does not compile.\n      let stderr = ''\n      try {\n        const r = await openscad(['-o', stlPath, scad], dir)\n        stderr = r.stderr\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        return { resolved: false, score: 0, detail: `compile failed: ${msg.slice(0, 300)}` }\n      }\n\n      // Render a PNG alongside (the artifact run-capsule animates). Non-fatal.\n      try {\n        await openscad(\n          ['-o', pngPath, '--imgsize=1100,850', '--camera=40,30,40,55,0,25,260', '--colorscheme=Tomorrow', scad],\n          dir,\n        )\n      } catch {\n        /* render is for the reel; geometry gate already has the STL */\n      }\n\n      let stl: string\n      try {\n        stl = await readFile(stlPath, 'utf8')\n      } catch {\n        return { resolved: false, score: 0, detail: 'compiled but produced no STL (empty geometry)' }\n      }\n      const volumes = Number(/Volumes:\\s*(\\d+)/.exec(stderr)?.[1] ?? '1')\n      const geo = measureStl(stl, volumes)\n      const { checks, score } = scoreGeometry(geo, spec)\n      const resolved = score === 1\n      const detail = JSON.stringify({\n        checks,\n        geo: {\n          triangles: geo.triangles,\n          volumes: geo.volumes,\n          bbox: { x: +geo.bbox.x.toFixed(1), y: +geo.bbox.y.toFixed(1), z: +geo.bbox.z.toFixed(1) },\n          hollowRatio: +(geo.solidVolume / geo.bboxVolume).toFixed(3),\n          topVsBaseFootprint: geo.baseFootprint ? +(geo.topFootprint / geo.baseFootprint).toFixed(3) : null,\n        },\n        renderPath: pngPath,\n        stlPath,\n      })\n      return { resolved, score, detail }\n    },\n  }\n}\n"],"mappings":";AAyBA,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,iBAAiB;AAC7C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AAoBxC,eAAe,SAAS,MAAgB,KAA0D;AAEhG,QAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,YAAY,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG;AAAA,IACtF;AAAA,IACA,WAAW,OAAO,OAAO;AAAA,IACzB,SAAS;AAAA,EACX,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO;AAC1B;AAeA,SAAS,WAAW,KAAa,mBAAqC;AACpE,QAAM,QAAyC,CAAC;AAChD,QAAM,OAA8F,CAAC;AACrG,QAAM,KAAK;AACX,MAAI;AACJ,QAAM,OAAwC,CAAC;AAC/C,SAAQ,IAAI,GAAG,KAAK,GAAG,EAAI,MAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/E,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK,QAAQ,KAAK,EAAG,MAAK,KAAK,CAAC,KAAK,CAAC,GAAI,KAAK,IAAI,CAAC,GAAI,KAAK,IAAI,CAAC,CAAE,CAAC;AAC7F,aAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAElC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,MAAM,CAAC,MAAgB,KAAK,IAAI,GAAG,CAAC;AAC1C,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,OAAO,IAAI,EAAE;AACnB,QAAM,IAAI,OAAO,QAAQ;AAGzB,QAAM,YAAY,CAAC,IAAY,OAAuB;AACpD,UAAM,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,KAAK,EAAE,CAAC,KAAK,OAAO,KAAK,CAAC;AAC/E,QAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,UAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,YAAQ,IAAI,EAAE,IAAI,IAAI,EAAE,MAAM,IAAI,EAAE,IAAI,IAAI,EAAE;AAAA,EAChD;AAGA,MAAI,OAAO;AACX,aAAW,CAAC,GAAG,GAAG,CAAC,KAAK,MAAM;AAC5B,YACE,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAChC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpC;AACA,QAAM,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,GAAG,OAAO,KAAK;AAC1E,SAAO;AAAA,IACL,WAAW,KAAK;AAAA,IAChB,SAAS;AAAA,IACT;AAAA,IACA,eAAe,UAAU,GAAG,GAAG;AAAA,IAC/B,cAAc,UAAU,MAAM,CAAC;AAAA,IAC/B,aAAa,KAAK,IAAI,IAAI,IAAI;AAAA,IAC9B,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,KAAK;AAAA,EAC1C;AACF;AAEA,SAAS,QAAQ,GAAW,CAAC,IAAI,EAAE,GAA8B;AAC/D,SAAO,KAAK,MAAM,KAAK;AACzB;AAGA,SAAS,cAAc,GAAa,MAAmE;AACrG,QAAM,SAAkC,CAAC;AACzC,MAAI,KAAK,QAAS,QAAO,UAAU,QAAQ,EAAE,SAAS,KAAK,OAAO;AAClE,MAAI,KAAK,gBAAgB,KAAM,QAAO,SAAS,EAAE,aAAa,KAAK;AACnE,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ,EAAE,KAAK,GAAG,KAAK,KAAK,CAAC;AAC9D,MAAI,KAAK,eAAe;AACtB,WAAO,cAAc,EAAE,gBAAgB,KAAK,EAAE,eAAe,KAAK,cAAc,EAAE;AACpF,MAAI,KAAK,eAAe,KAAM,QAAO,SAAS,EAAE,cAAc,KAAK,cAAc,EAAE;AACnF,QAAM,OAAO,OAAO,OAAO,MAAM;AACjC,QAAM,QAAQ,KAAK,SAAS,KAAK,OAAO,OAAO,EAAE,SAAS,KAAK,SAAS;AACxE,SAAO,EAAE,QAAQ,MAAM;AACzB;AAMA,SAAS,aAAa,GAAW,GAAW,IAAY,IAAY,IAAY,IAAI,GAAW;AAC7F,SAAO;AAAA,IACL,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUjD,KAAK;AACP;AAOA,SAAS,aAAa,GAIpB;AACA,QAAM,EAAE,IAAI,MAAM,GAAG,GAAG,IAAI,IAAI,GAAG,IAAI;AACvC,QAAM,YAAY,KAAK;AACvB,QAAM,SAAS,YAAY;AAC3B,QAAM,IAAI,KAAK;AACf,QAAM,OAAgB;AAAA,IACpB,SAAS,CAAC,GAAG,KAAK,IAAI,CAAC;AAAA,IACvB,MAAM;AAAA,MACJ,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,CAAC;AAAA,MAC5B,GAAG,CAAC,EAAE,IAAI,IAAI,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC;AAAA,MACjC,GAAG,CAAC,EAAE,SAAS,IAAI,GAAG,GAAG,SAAS,KAAK,GAAG,CAAC;AAAA,IAC7C;AAAA,IACA,cAAc;AAAA,IACd,aAAa;AAAA,IACb,aAAa;AAAA,EACf;AACA,QAAM,SAAS;AAAA,IACb,6BAA6B,IAAI;AAAA,IACjC;AAAA,IACA,uBAAuB,CAAC,WAAW,CAAC;AAAA,IACpC,KAAK,IAAI,KAAK,EAAE,gCAAgC,SAAS,WAAW,6BAA6B,EAAE;AAAA,IACnG;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACX,SAAO,EAAE,IAAI,QAAQ,MAAM,EAAE,MAAM,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE;AAC5E;AAIA,IAAM,mBAAkH;AAAA,EACtH,EAAE,IAAI,WAAW,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACrF,EAAE,IAAI,SAAS,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,YAAY,MAAM,gCAAgC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5F,EAAE,IAAI,QAAQ,MAAM,oBAAoB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC5E,EAAE,IAAI,aAAa,MAAM,qBAAqB,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACnF,EAAE,IAAI,UAAU,MAAM,0CAA0C,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACpG,EAAE,IAAI,aAAa,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EACtF,EAAE,IAAI,mBAAmB,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC7F,EAAE,IAAI,aAAa,MAAM,kCAAkC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AAAA,EAC/F,EAAE,IAAI,cAAc,MAAM,wCAAwC,GAAG,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG;AACxG;AAIA,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMvB,KAAK;AAIP,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOlB,KAAK;AAEP,IAAM,QAAkE;AAAA,EACtE;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,EAAE;AAAA,QACf,MAAM,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAChD,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWV,KAAK;AAAA,IACH;AAAA,EACF;AAAA,EACA,GAAG,iBAAiB,IAAI,YAAY;AAAA,EACpC;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE;AAAA,QAC/C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,QACb,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,QACJ,SAAS,CAAC,GAAG,CAAC;AAAA,QACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE;AAAA,QAC9C,cAAc;AAAA,QACd,aAAa;AAAA,MACf;AAAA,MACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMV,KAAK;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM,cAAc,YAAY,CAAC,MAAM,YAAY,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MACtF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,gCAAgC,GAAG;AAAA;AAAA,QAGrC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,QAAQ;AACZ,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,EAAE,KAA2C,EAAE;AAAA,IAC7H;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK;AAClB,aAAO,MAAM;AAAA,IACf;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAQ,KAAK,SAAoC;AACvD,YAAM,MAAM,SAAS,KAAK;AAC1B,UAAI,CAAC,IAAK,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AAEvE,YAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,MAAM,CAAC;AAChD,YAAM,OAAO,KAAK,KAAK,YAAY;AACnC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,KAAK,KAAK,WAAW;AACrC,YAAM,UAAU,MAAM,GAAG;AAGzB,UAAI,SAAS;AACb,UAAI;AACF,cAAM,IAAI,MAAM,SAAS,CAAC,MAAM,SAAS,IAAI,GAAG,GAAG;AACnD,iBAAS,EAAE;AAAA,MACb,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,mBAAmB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,MACrF;AAGA,UAAI;AACF,cAAM;AAAA,UACJ,CAAC,MAAM,SAAS,sBAAsB,iCAAiC,0BAA0B,IAAI;AAAA,UACrG;AAAA,QACF;AAAA,MACF,QAAQ;AAAA,MAER;AAEA,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,SAAS,SAAS,MAAM;AAAA,MACtC,QAAQ;AACN,eAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gDAAgD;AAAA,MAC9F;AACA,YAAM,UAAU,OAAO,mBAAmB,KAAK,MAAM,IAAI,CAAC,KAAK,GAAG;AAClE,YAAM,MAAM,WAAW,KAAK,OAAO;AACnC,YAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,KAAK,IAAI;AACjD,YAAM,WAAW,UAAU;AAC3B,YAAM,SAAS,KAAK,UAAU;AAAA,QAC5B;AAAA,QACA,KAAK;AAAA,UACH,WAAW,IAAI;AAAA,UACf,SAAS,IAAI;AAAA,UACb,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,EAAE,QAAQ,CAAC,EAAE;AAAA,UACxF,aAAa,EAAE,IAAI,cAAc,IAAI,YAAY,QAAQ,CAAC;AAAA,UAC1D,oBAAoB,IAAI,gBAAgB,EAAE,IAAI,eAAe,IAAI,eAAe,QAAQ,CAAC,IAAI;AAAA,QAC/F;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,MACF,CAAC;AACD,aAAO,EAAE,UAAU,OAAO,OAAO;AAAA,IACnC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-5H5XV76F.js

@@ -1,240 +0,0 @@
-// src/benchmarks/humaneval.ts
-import { execFile } from "child_process";
-import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
-import { tmpdir } from "os";
-import { join } from "path";
-import { gunzipSync } from "zlib";
-var humanevalUrl = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz";
-var dockerImage = "python:3.12-slim";
-var dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 2e4);
-async function loadHumanEval(limit, offset = 0) {
-  const localGz = process.env.HUMANEVAL_GZ;
-  let gz;
-  if (localGz) {
-    gz = readFileSync(localGz);
-  } else {
-    const res = await fetch(humanevalUrl);
-    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`);
-    gz = Buffer.from(await res.arrayBuffer());
-  }
-  const text = gunzipSync(gz).toString("utf8");
-  const tasks = [];
-  for (const line of text.split("\n")) {
-    if (line.trim() === "") continue;
-    const d = JSON.parse(line);
-    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {
-      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`);
-    }
-    tasks.push({
-      taskId: d.task_id,
-      prompt: d.prompt,
-      test: d.test,
-      entryPoint: d.entry_point,
-      ...d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}
-    });
-  }
-  if (tasks.length === 0) throw new Error("HumanEval parsed to 0 tasks");
-  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`);
-  return tasks.slice(offset, offset + limit);
-}
-var solveInstruction = "Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.";
-function basePrompt(task) {
-  return `${solveInstruction}
-
-\`\`\`python
-${task.prompt}\`\`\``;
-}
-function extractCode(reply) {
-  const fenced = reply.match(/```(?:python|py)?\s*\n([\s\S]*?)```/i);
-  if (fenced && typeof fenced[1] === "string") return fenced[1].trim();
-  return reply.trim();
-}
-function buildProgram(task, candidate) {
-  return `${task.prompt}
-${candidate}
-
-${task.test}
-
-check(${task.entryPoint})
-`;
-}
-var dockerRunSeq = 0;
-function runPythonProgram(program, timeoutMs = dockerTimeoutMs) {
-  const dir = mkdtempSync(join(tmpdir(), "hev-"));
-  chmodSync(dir, 493);
-  writeFileSync(join(dir, "p.py"), program, { mode: 420 });
-  const name = `hev-${process.pid}-${dockerRunSeq++}`;
-  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`;
-  return new Promise((resolvePromise, reject) => {
-    let settled = false;
-    const cleanup = () => {
-      rmSync(dir, { recursive: true, force: true });
-      execFile("docker", ["rm", "-f", name], () => {
-      });
-    };
-    const finish = (res) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(backstop);
-      cleanup();
-      resolvePromise(res);
-    };
-    const fail = (e) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(backstop);
-      cleanup();
-      reject(e);
-    };
-    const outerTimeoutMs = timeoutMs + 3e3;
-    const backstop = setTimeout(
-      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3e3}ms`)),
-      outerTimeoutMs + 3e3
-    );
-    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1e3));
-    execFile(
-      "docker",
-      [
-        "run",
-        "--rm",
-        "--pull=never",
-        "--name",
-        name,
-        "--network=none",
-        "--cpus=1",
-        "--memory=512m",
-        "--pids-limit=64",
-        "--cap-drop=ALL",
-        "--security-opt=no-new-privileges",
-        "--read-only",
-        "--tmpfs",
-        "/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777",
-        "--user",
-        "65534:65534",
-        "-v",
-        `${dir}:/w:ro`,
-        "-w",
-        "/w",
-        dockerImage,
-        "sh",
-        "-c",
-        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf "%s\\n" "$1"; exec timeout -s KILL "$2" python /w/p.py',
-        "agent-runtime-checker",
-        startMarker,
-        `${inContainerSeconds}s`
-      ],
-      { timeout: outerTimeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024 },
-      (err, stdout, stderr) => {
-        const markerLine = `${startMarker}
-`;
-        const candidateStarted = stdout?.startsWith(markerLine) === true;
-        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : stdout ?? "";
-        if (err) {
-          const e = err;
-          if (e.code === "ENOENT") {
-            fail(new Error("docker binary not found on PATH \u2014 cannot run the deployable checker"));
-            return;
-          }
-          if (e.killed) {
-            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`));
-            return;
-          }
-          if (!candidateStarted) {
-            if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(
-              stderr
-            )) {
-              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
-              return;
-            }
-            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
-              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
-              return;
-            }
-            fail(
-              new Error(
-                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`
-              )
-            );
-            return;
-          }
-          const exitCode = typeof e.code === "number" ? e.code : 1;
-          finish({
-            exitCode,
-            stdout: candidateStdout,
-            stderr: stderr ?? ""
-          });
-          return;
-        }
-        if (!candidateStarted) {
-          fail(new Error("docker checker exited without starting the candidate"));
-          return;
-        }
-        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? "" });
-      }
-    );
-  });
-}
-async function runChecker(task, candidate) {
-  const result = await runPythonProgram(buildProgram(task, candidate));
-  return result.exitCode === 0 ? { pass: 1 } : { pass: 0, detail: result.stderr.slice(-600) || "timed out (no output)" };
-}
-function toBenchTask(t) {
-  const metadata = {
-    promptHeader: t.prompt,
-    test: t.test,
-    entryPoint: t.entryPoint,
-    ...t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}
-  };
-  return { id: t.taskId, prompt: basePrompt(t), metadata };
-}
-function taskFromMeta(task) {
-  const m = task.metadata;
-  if (!m?.promptHeader || !m.test || !m.entryPoint) {
-    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`);
-  }
-  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint };
-}
-function createHumanEvalAdapter() {
-  return {
-    name: "humaneval",
-    async preflight() {
-      await new Promise((resolve, reject) => {
-        execFile("docker", ["version", "--format", "{{.Server.Version}}"], (err) => {
-          if (err) reject(new Error("HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)"));
-          else resolve();
-        });
-      });
-      await new Promise((resolve, reject) => {
-        execFile("docker", ["image", "inspect", dockerImage], (err) => {
-          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`));
-          else resolve();
-        });
-      });
-    },
-    async loadTasks(opts) {
-      const offset = Number(process.env.OFFSET ?? 0);
-      const all = await loadHumanEval(opts?.ids ? 164 : opts?.limit ?? 8, offset);
-      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all;
-      return picked.map(toBenchTask);
-    },
-    async judge(task, artifact) {
-      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact));
-      return { resolved: pass === 1, score: pass, detail: pass === 1 ? "tests passed" : "tests failed" };
-    },
-    async goldArtifact(task) {
-      const m = task.metadata;
-      const sol = m?.canonicalSolution;
-      return sol ? `${m.promptHeader}${sol}` : void 0;
-    }
-  };
-}
-
-export {
-  loadHumanEval,
-  basePrompt,
-  extractCode,
-  runPythonProgram,
-  runChecker,
-  createHumanEvalAdapter
-};
-//# sourceMappingURL=chunk-5H5XV76F.js.map

package/dist/chunk-5H5XV76F.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/humaneval.ts"],"sourcesContent":["/**\n * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the\n * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout\n * through the `Supervisor` that self-corrects across rounds, vs blind\n * random@k. This is the experiment `humaneval-gate.mts` names as \"the next one\" —\n * the gate measures SELECTION over stateless single completions; this measures\n * whether observe→steer (self-correction) beats blind compute at equal k.\n *\n * Worker artifact = the model's reply (a Python function, fenced or raw). The\n * DETERMINISTIC judge runs the candidate against the task's own `test` in an\n * isolated `--network=none` python container — exit 0 = pass. No gold\n * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only\n * to self-verify the judge before spending tokens.\n *\n * The primitives (loader / extractor / Docker checker) live here and are reused by\n * `humaneval-gate.mts` — one home, no duplication.\n */\n\nimport { execFile } from 'node:child_process'\nimport { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { gunzipSync } from 'node:zlib'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'\nconst dockerImage = 'python:3.12-slim'\nconst dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)\n\nexport interface HumanEvalTask {\n  taskId: string\n  prompt: string\n  test: string\n  entryPoint: string\n  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */\n  canonicalSolution?: string\n}\n\n/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch\n *  or a malformed line — a silently-short task set would poison the gate. `offset`\n *  selects a deeper slice (the later tasks are harder) so the worker has a\n *  correctable middle band rather than a saturated easy prefix. */\nexport async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {\n  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits\n  // (429) under repeated runs. Fall back to the network fetch when unset.\n  const localGz = process.env.HUMANEVAL_GZ\n  let gz: Buffer\n  if (localGz) {\n    gz = readFileSync(localGz)\n  } else {\n    const res = await fetch(humanevalUrl)\n    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)\n    gz = Buffer.from(await res.arrayBuffer())\n  }\n  const text = gunzipSync(gz).toString('utf8')\n  const tasks: HumanEvalTask[] = []\n  for (const line of text.split('\\n')) {\n    if (line.trim() === '') continue\n    const d = JSON.parse(line) as {\n      task_id?: string\n      prompt?: string\n      test?: string\n      entry_point?: string\n      canonical_solution?: string\n    }\n    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {\n      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)\n    }\n    tasks.push({\n      taskId: d.task_id,\n      prompt: d.prompt,\n      test: d.test,\n      entryPoint: d.entry_point,\n      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),\n    })\n  }\n  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')\n  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)\n  return tasks.slice(offset, offset + limit)\n}\n\nconst solveInstruction =\n  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'\n\nexport function basePrompt(task: HumanEvalTask): string {\n  return `${solveInstruction}\\n\\n\\`\\`\\`python\\n${task.prompt}\\`\\`\\``\n}\n\n/** Extract the function source from a model reply: prefer a fenced ```python (or\n *  bare ```) block, else fall back to the raw text. The deployable program adds the\n *  prompt header (imports + signature context), so a candidate that returns only a\n *  body still runs; a candidate that re-defines the function shadows the header. */\nexport function extractCode(reply: string): string {\n  const fenced = reply.match(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i)\n  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()\n  return reply.trim()\n}\n\n/** The deployable test program: the prompt header (imports + signature/docstring the\n *  model was given), then the candidate (its def shadows the header's stub), then the\n *  task's own check() suite and the call. No gold solution anywhere. */\nfunction buildProgram(task: HumanEvalTask, candidate: string): string {\n  return `${task.prompt}\\n${candidate}\\n\\n${task.test}\\n\\ncheck(${task.entryPoint})\\n`\n}\n\nexport interface CheckResult {\n  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */\n  pass: number\n  /** On failure: the interpreter stderr tail (traceback / failing assertion). The\n   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */\n  detail?: string\n}\n\nexport interface PythonProgramResult {\n  exitCode: number\n  stdout: string\n  stderr: string\n}\n\n/** Run one candidate's deployable test program in an isolated container:\n *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.\n *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image\n *  unavailable) is NOT a test failure — it throws so the harness fails loud rather\n *  than scoring every candidate 0 from a broken checker. */\nlet dockerRunSeq = 0\n\n/** Run arbitrary Python in the same isolated container used by the HumanEval\n * checker. This is also the execution-tool primitive for experiments that let\n * a model test its own snippets; model-written code must never run on the host. */\nexport function runPythonProgram(\n  program: string,\n  timeoutMs = dockerTimeoutMs,\n): Promise<PythonProgramResult> {\n  const dir = mkdtempSync(join(tmpdir(), 'hev-'))\n  // Rootless Docker maps container root to a subordinate host uid that cannot\n  // traverse mkdtemp's default 0700 directory. The directory contains only the\n  // disposable candidate script and is mounted read-only into the container.\n  chmodSync(dir, 0o755)\n  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })\n  // Unique container name so we can force-reap it regardless of the docker client's state.\n  const name = `hev-${process.pid}-${dockerRunSeq++}`\n  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`\n  return new Promise<PythonProgramResult>((resolvePromise, reject) => {\n    let settled = false\n    const cleanup = () => {\n      rmSync(dir, { recursive: true, force: true })\n      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung\n      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;\n      // the name is unique, so no reuse race).\n      execFile('docker', ['rm', '-f', name], () => {})\n    }\n    const finish = (res: PythonProgramResult) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      resolvePromise(res)\n    }\n    const fail = (e: Error) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      reject(e)\n    }\n    // Candidate timeouts are enforced inside the container. If the outer Docker\n    // client or daemon misses this larger deadline, that is infrastructure failure,\n    // not a wrong answer.\n    const outerTimeoutMs = timeoutMs + 3_000\n    const backstop = setTimeout(\n      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),\n      outerTimeoutMs + 3_000,\n    )\n    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))\n    execFile(\n      'docker',\n      [\n        'run',\n        '--rm',\n        '--pull=never',\n        '--name',\n        name,\n        '--network=none',\n        '--cpus=1',\n        '--memory=512m',\n        '--pids-limit=64',\n        '--cap-drop=ALL',\n        '--security-opt=no-new-privileges',\n        '--read-only',\n        '--tmpfs',\n        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',\n        '--user',\n        '65534:65534',\n        '-v',\n        `${dir}:/w:ro`,\n        '-w',\n        '/w',\n        dockerImage,\n        'sh',\n        '-c',\n        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf \"%s\\\\n\" \"$1\"; exec timeout -s KILL \"$2\" python /w/p.py',\n        'agent-runtime-checker',\n        startMarker,\n        `${inContainerSeconds}s`,\n      ],\n      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },\n      (err, stdout, stderr) => {\n        const markerLine = `${startMarker}\\n`\n        const candidateStarted = stdout?.startsWith(markerLine) === true\n        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')\n        if (err) {\n          const e = err as Error & { killed?: boolean; code?: number | string }\n          if (e.code === 'ENOENT') {\n            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))\n            return\n          }\n          if (e.killed) {\n            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))\n            return\n          }\n          if (!candidateStarted) {\n            if (\n              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(\n                stderr,\n              )\n            ) {\n              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {\n              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            fail(\n              new Error(\n                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,\n              ),\n            )\n            return\n          }\n          const exitCode = typeof e.code === 'number' ? e.code : 1\n          finish({\n            exitCode,\n            stdout: candidateStdout,\n            stderr: stderr ?? '',\n          })\n          return\n        }\n        if (!candidateStarted) {\n          fail(new Error('docker checker exited without starting the candidate'))\n          return\n        }\n        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })\n      },\n    )\n  })\n}\n\nexport async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {\n  const result = await runPythonProgram(buildProgram(task, candidate))\n  return result.exitCode === 0\n    ? { pass: 1 }\n    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }\n}\n\n/** A HumanEval task carries its checker inputs in metadata so the deterministic\n *  judge can rebuild the deployable program from a `BenchTask` alone. */\ninterface HumanEvalMeta extends Record<string, unknown> {\n  promptHeader: string\n  test: string\n  entryPoint: string\n  canonicalSolution?: string\n}\n\nfunction toBenchTask(t: HumanEvalTask): BenchTask {\n  const metadata: HumanEvalMeta = {\n    promptHeader: t.prompt,\n    test: t.test,\n    entryPoint: t.entryPoint,\n    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),\n  }\n  return { id: t.taskId, prompt: basePrompt(t), metadata }\n}\n\nfunction taskFromMeta(task: BenchTask): HumanEvalTask {\n  const m = task.metadata as HumanEvalMeta | undefined\n  if (!m?.promptHeader || !m.test || !m.entryPoint) {\n    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)\n  }\n  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }\n}\n\n/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle\n *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */\nexport function createHumanEvalAdapter(): BenchmarkAdapter {\n  return {\n    name: 'humaneval',\n    async preflight() {\n      // Prove the daemon and exact local image before any model call. Scoring uses\n      // --pull=never so a registry or cold pull cannot become a candidate outcome.\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {\n          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))\n          else resolve()\n        })\n      })\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['image', 'inspect', dockerImage], (err) => {\n          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))\n          else resolve()\n        })\n      })\n    },\n    async loadTasks(opts?: LoadOptions) {\n      const offset = Number(process.env.OFFSET ?? 0)\n      // Pull a generous window when filtering by id, else exactly `limit` from offset.\n      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)\n      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all\n      return picked.map(toBenchTask)\n    },\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))\n      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }\n    },\n    async goldArtifact(task: BenchTask) {\n      const m = task.metadata as HumanEvalMeta | undefined\n      const sol = m?.canonicalSolution\n      // Return the COMPLETE function (signature header + canonical body), i.e. what a\n      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose\n      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its\n      // leading indent and breaks it, so a body-only gold fails its own judge. A full\n      // def starts at column 0, trims safely, and self-verifies.\n      return sol ? `${m!.promptHeader}${sol}` : undefined\n    },\n  }\n}\n"],"mappings":";AAkBA,SAAS,gBAAgB;AACzB,SAAS,WAAW,aAAa,cAAc,QAAQ,qBAAqB;AAC5E,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAG3B,IAAM,eAAe;AACrB,IAAM,cAAc;AACpB,IAAM,kBAAkB,OAAO,QAAQ,IAAI,qBAAqB,GAAK;AAerE,eAAsB,cAAc,OAAe,SAAS,GAA6B;AAGvF,QAAM,UAAU,QAAQ,IAAI;AAC5B,MAAI;AACJ,MAAI,SAAS;AACX,SAAK,aAAa,OAAO;AAAA,EAC3B,OAAO;AACL,UAAM,MAAM,MAAM,MAAM,YAAY;AACpC,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,KAAK,YAAY,EAAE;AAClF,SAAK,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC;AAAA,EAC1C;AACA,QAAM,OAAO,WAAW,EAAE,EAAE,SAAS,MAAM;AAC3C,QAAM,QAAyB,CAAC;AAChC,aAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,QAAI,KAAK,KAAK,MAAM,GAAI;AACxB,UAAM,IAAI,KAAK,MAAM,IAAI;AAOzB,QAAI,CAAC,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,aAAa;AACxD,YAAM,IAAI,MAAM,+BAA+B,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrE;AACA,UAAM,KAAK;AAAA,MACT,QAAQ,EAAE;AAAA,MACV,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,YAAY,EAAE;AAAA,MACd,GAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,IAAI,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B;AACrE,MAAI,UAAU,MAAM,OAAQ,OAAM,IAAI,MAAM,UAAU,MAAM,oBAAoB,MAAM,MAAM,EAAE;AAC9F,SAAO,MAAM,MAAM,QAAQ,SAAS,KAAK;AAC3C;AAEA,IAAM,mBACJ;AAEK,SAAS,WAAW,MAA6B;AACtD,SAAO,GAAG,gBAAgB;AAAA;AAAA;AAAA,EAAqB,KAAK,MAAM;AAC5D;AAMO,SAAS,YAAY,OAAuB;AACjD,QAAM,SAAS,MAAM,MAAM,sCAAsC;AACjE,MAAI,UAAU,OAAO,OAAO,CAAC,MAAM,SAAU,QAAO,OAAO,CAAC,EAAE,KAAK;AACnE,SAAO,MAAM,KAAK;AACpB;AAKA,SAAS,aAAa,MAAqB,WAA2B;AACpE,SAAO,GAAG,KAAK,MAAM;AAAA,EAAK,SAAS;AAAA;AAAA,EAAO,KAAK,IAAI;AAAA;AAAA,QAAa,KAAK,UAAU;AAAA;AACjF;AAqBA,IAAI,eAAe;AAKZ,SAAS,iBACd,SACA,YAAY,iBACkB;AAC9B,QAAM,MAAM,YAAY,KAAK,OAAO,GAAG,MAAM,CAAC;AAI9C,YAAU,KAAK,GAAK;AACpB,gBAAc,KAAK,KAAK,MAAM,GAAG,SAAS,EAAE,MAAM,IAAM,CAAC;AAEzD,QAAM,OAAO,OAAO,QAAQ,GAAG,IAAI,cAAc;AACjD,QAAM,cAAc,qCAAqC,IAAI;AAC7D,SAAO,IAAI,QAA6B,CAAC,gBAAgB,WAAW;AAClE,QAAI,UAAU;AACd,UAAM,UAAU,MAAM;AACpB,aAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAI5C,eAAS,UAAU,CAAC,MAAM,MAAM,IAAI,GAAG,MAAM;AAAA,MAAC,CAAC;AAAA,IACjD;AACA,UAAM,SAAS,CAAC,QAA6B;AAC3C,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,qBAAe,GAAG;AAAA,IACpB;AACA,UAAM,OAAO,CAAC,MAAa;AACzB,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,aAAO,CAAC;AAAA,IACV;AAIA,UAAM,iBAAiB,YAAY;AACnC,UAAM,WAAW;AAAA,MACf,MAAM,KAAK,IAAI,MAAM,wCAAwC,iBAAiB,GAAK,IAAI,CAAC;AAAA,MACxF,iBAAiB;AAAA,IACnB;AACA,UAAM,qBAAqB,KAAK,IAAI,GAAG,KAAK,KAAK,YAAY,GAAK,CAAC;AACnE;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,GAAG;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,kBAAkB;AAAA,MACvB;AAAA,MACA,EAAE,SAAS,gBAAgB,YAAY,WAAW,WAAW,IAAI,OAAO,KAAK;AAAA,MAC7E,CAAC,KAAK,QAAQ,WAAW;AACvB,cAAM,aAAa,GAAG,WAAW;AAAA;AACjC,cAAM,mBAAmB,QAAQ,WAAW,UAAU,MAAM;AAC5D,cAAM,kBAAkB,mBAAmB,OAAO,MAAM,WAAW,MAAM,IAAK,UAAU;AACxF,YAAI,KAAK;AACP,gBAAM,IAAI;AACV,cAAI,EAAE,SAAS,UAAU;AACvB,iBAAK,IAAI,MAAM,0EAAqE,CAAC;AACrF;AAAA,UACF;AACA,cAAI,EAAE,QAAQ;AACZ,iBAAK,IAAI,MAAM,+BAA+B,cAAc,kBAAkB,CAAC;AAC/E;AAAA,UACF;AACA,cAAI,CAAC,kBAAkB;AACrB,gBACE,8GAA8G;AAAA,cAC5G;AAAA,YACF,GACA;AACA,mBAAK,IAAI,MAAM,8BAA8B,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AACpE;AAAA,YACF;AACA,gBAAI,0EAA0E,KAAK,MAAM,GAAG;AAC1F,mBAAK,IAAI,MAAM,gBAAgB,WAAW,iBAAiB,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AAClF;AAAA,YACF;AACA;AAAA,cACE,IAAI;AAAA,gBACF,+CAA+C,OAAO,MAAM,GAAG,GAAG,KAAK,EAAE,OAAO;AAAA,cAClF;AAAA,YACF;AACA;AAAA,UACF;AACA,gBAAM,WAAW,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;AACvD,iBAAO;AAAA,YACL;AAAA,YACA,QAAQ;AAAA,YACR,QAAQ,UAAU;AAAA,UACpB,CAAC;AACD;AAAA,QACF;AACA,YAAI,CAAC,kBAAkB;AACrB,eAAK,IAAI,MAAM,sDAAsD,CAAC;AACtE;AAAA,QACF;AACA,eAAO,EAAE,UAAU,GAAG,QAAQ,iBAAiB,QAAQ,UAAU,GAAG,CAAC;AAAA,MACvE;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,eAAsB,WAAW,MAAqB,WAAyC;AAC7F,QAAM,SAAS,MAAM,iBAAiB,aAAa,MAAM,SAAS,CAAC;AACnE,SAAO,OAAO,aAAa,IACvB,EAAE,MAAM,EAAE,IACV,EAAE,MAAM,GAAG,QAAQ,OAAO,OAAO,MAAM,IAAI,KAAK,wBAAwB;AAC9E;AAWA,SAAS,YAAY,GAA6B;AAChD,QAAM,WAA0B;AAAA,IAC9B,cAAc,EAAE;AAAA,IAChB,MAAM,EAAE;AAAA,IACR,YAAY,EAAE;AAAA,IACd,GAAI,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC;AAAA,EAC1E;AACA,SAAO,EAAE,IAAI,EAAE,QAAQ,QAAQ,WAAW,CAAC,GAAG,SAAS;AACzD;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,IAAI,KAAK;AACf,MAAI,CAAC,GAAG,gBAAgB,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY;AAChD,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,2BAA2B;AAAA,EAC7E;AACA,SAAO,EAAE,QAAQ,KAAK,IAAI,QAAQ,EAAE,cAAc,MAAM,EAAE,MAAM,YAAY,EAAE,WAAW;AAC3F;AAIO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,YAAY;AAGhB,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,WAAW,YAAY,qBAAqB,GAAG,CAAC,QAAQ;AAC1E,cAAI,IAAK,QAAO,IAAI,MAAM,kFAAkF,CAAC;AAAA,cACxG,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AACD,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,SAAS,WAAW,WAAW,GAAG,CAAC,QAAQ;AAC7D,cAAI,IAAK,QAAO,IAAI,MAAM,iDAAiD,WAAW,EAAE,CAAC;AAAA,cACpF,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,IACA,MAAM,UAAU,MAAoB;AAClC,YAAM,SAAS,OAAO,QAAQ,IAAI,UAAU,CAAC;AAE7C,YAAM,MAAM,MAAM,cAAc,MAAM,MAAM,MAAO,MAAM,SAAS,GAAI,MAAM;AAC5E,YAAM,SAAS,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,SAAS,EAAE,MAAM,CAAC,IAAI;AAC7E,aAAO,OAAO,IAAI,WAAW;AAAA,IAC/B;AAAA,IACA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,KAAK,IAAI,MAAM,WAAW,aAAa,IAAI,GAAG,YAAY,QAAQ,CAAC;AAC3E,aAAO,EAAE,UAAU,SAAS,GAAG,OAAO,MAAM,QAAQ,SAAS,IAAI,iBAAiB,eAAe;AAAA,IACnG;AAAA,IACA,MAAM,aAAa,MAAiB;AAClC,YAAM,IAAI,KAAK;AACf,YAAM,MAAM,GAAG;AAMf,aAAO,MAAM,GAAG,EAAG,YAAY,GAAG,GAAG,KAAK;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-7GRVHU22.js

@@ -1,208 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/finresearchbench.ts
-import { readFile, stat } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "finresearchbench.json");
-var dataFile = () => process.env.FINRESEARCHBENCH_DATA_FILE;
-function routerConfig() {
-  const key = process.env.TANGLE_API_KEY;
-  if (!key) throw new Error("TANGLE_API_KEY is required for FinResearchBench live LLM judging");
-  return {
-    baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
-    key,
-    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? "deepseek-v4-flash"
-  };
-}
-async function assertReadable(path, label) {
-  try {
-    await stat(path);
-  } catch (err) {
-    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function readRecords(raw) {
-  const trimmed = raw.trim();
-  if (trimmed.startsWith("[")) return JSON.parse(trimmed);
-  return trimmed.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
-}
-function assertLiveJudgeFields(records, source) {
-  const missing = records.filter((row) => !row.judge_system_prompt || !row.judge_prompt_template).map((row) => row.id);
-  if (missing.length > 0) {
-    throw new Error(
-      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(", ")}. Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.`
-    );
-  }
-}
-function rowToTask(row, fixturesMode) {
-  const referenceAnswer = row.reference_answer ?? "";
-  const referenceReport = row.reference_report ?? referenceAnswer;
-  const meta = {
-    id: row.id,
-    category: row.category ?? "unknown",
-    question: row.question,
-    referenceAnswer,
-    referenceReport,
-    logicTree: row.logic_tree ?? null,
-    rubric: row.rubric ?? null,
-    judgeSystemPrompt: row.judge_system_prompt,
-    judgePromptTemplate: row.judge_prompt_template,
-    scoring: fixturesMode ? "fixture-exact-reference" : "official-logic-tree-judge"
-  };
-  return {
-    id: row.id,
-    split: row.category,
-    prompt: [
-      "Complete this FinResearchBench financial research task.",
-      "Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.",
-      "",
-      row.question
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.question !== "string") {
-    throw new Error(`FinResearchBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts, fixturesMode) {
-  let tasks = rows.map((row) => rowToTask(row, fixturesMode));
-  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const records = readRecords(await readFile(FIXTURES, "utf8"));
-  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 \u2014 loading ${records.length} adapter fixtures`);
-  return selectRows(records, opts, true);
-}
-async function loadOfficialTasks(path, opts) {
-  const records = readRecords(await readFile(path, "utf8"));
-  assertLiveJudgeFields(records, path);
-  return selectRows(records, opts, false);
-}
-function fillTemplate(meta, response) {
-  const template = meta.judgePromptTemplate;
-  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`);
-  return template.replaceAll("{question}", meta.question).replaceAll("{response}", response).replaceAll("{reference_answer}", meta.referenceAnswer).replaceAll("{reference_report}", meta.referenceReport).replaceAll("{logic_tree}", JSON.stringify(meta.logicTree, null, 2)).replaceAll("{rubric}", JSON.stringify(meta.rubric, null, 2));
-}
-function parseJudgeScore(content) {
-  const candidates = [];
-  for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
-  for (const m of content.matchAll(/\{[\s\S]*?\}/g)) candidates.push(m[0]);
-  candidates.push(content.trim());
-  for (const candidate of candidates) {
-    let parsed;
-    try {
-      parsed = JSON.parse(candidate);
-    } catch {
-      continue;
-    }
-    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score;
-    const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
-    if (!Number.isFinite(n)) continue;
-    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`);
-    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100;
-    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`);
-    return { score, raw: parsed };
-  }
-  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`);
-}
-async function runOfficialJudge(meta, response) {
-  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`);
-  const router = routerConfig();
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [
-        { role: "system", content: meta.judgeSystemPrompt },
-        { role: "user", content: fillTemplate(meta, response) }
-      ]
-    })
-  });
-  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  const body = await res.json();
-  const content = body.choices?.[0]?.message?.content;
-  if (typeof content !== "string") throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
-  const { score, raw } = parseJudgeScore(content);
-  return {
-    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
-    score,
-    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw })
-  };
-}
-function normalizeText(value) {
-  return value.toLowerCase().replace(/\s+/g, " ").trim();
-}
-function scoreFixture(meta, artifact) {
-  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport);
-  const response = normalizeText(artifact);
-  const score = answer.length > 0 && response.includes(answer) ? 1 : 0;
-  return {
-    resolved: score === 1,
-    score,
-    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category })
-  };
-}
-function createFinResearchBenchAdapter() {
-  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === "1";
-  return {
-    name: "finresearchbench",
-    async preflight() {
-      if (fixturesMode) {
-        await assertReadable(FIXTURES, "fixture file");
-        return;
-      }
-      const file = dataFile();
-      if (!file) {
-        throw new Error(
-          "FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl."
-        );
-      }
-      routerConfig();
-      await assertReadable(file, "data file");
-      await loadOfficialTasks(file, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const file = dataFile();
-      if (!file) throw new Error("FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows");
-      return loadOfficialTasks(file, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return meta.referenceReport || meta.referenceAnswer || void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      if (fixturesMode) return scoreFixture(meta, artifact);
-      if (artifact.trim().length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: "empty answer" })
-        };
-      }
-      return runOfficialJudge(meta, artifact);
-    }
-  };
-}
-
-export {
-  createFinResearchBenchAdapter
-};
-//# sourceMappingURL=chunk-7GRVHU22.js.map

package/dist/chunk-7GRVHU22.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,UAAU,YAAY;AAC/B,SAAS,YAAY;AAIrB,IAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,eAAgE;AACvE,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,kEAAkE;AAC5F,SAAO;AAAA,IACL,SAAS,QAAQ,IAAI,eAAe;AAAA,IACpC;AAAA,IACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;AAAA,EAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;AACxE,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,6BAA6B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;AACrD,QAAM,UAAU,IAAI,KAAK;AACzB,MAAI,QAAQ,WAAW,GAAG,EAAG,QAAO,KAAK,MAAM,OAAO;AACtD,SAAO,QACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;AAC1F,QAAM,UAAU,QACb,OAAO,CAAC,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,EACtE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtB,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,mCAAmC,MAAM,uCAAuC,QAAQ,MAAM,IAAI,QAAQ,MAAM,YAAY,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAE5J;AAAA,EACF;AACF;AAEA,SAAS,UAAU,KAAwB,cAAkC;AAC3E,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,OAAwB;AAAA,IAC5B,IAAI,IAAI;AAAA,IACR,UAAU,IAAI,YAAY;AAAA,IAC1B,UAAU,IAAI;AAAA,IACd;AAAA,IACA;AAAA,IACA,WAAW,IAAI,cAAc;AAAA,IAC7B,QAAQ,IAAI,UAAU;AAAA,IACtB,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,IACzB,SAAS,eAAe,4BAA4B;AAAA,EACtD;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,IACN,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAkC;AAClD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAAU;AAC1C,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,wDAAmD;AAAA,EACrG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;AACpG,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,YAAY,CAAC;AAC1D,MAAI,KAAK,MAAO,SAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AACxE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,EAAE;AACpG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ,KAAK,iEAA4D,QAAQ,MAAM,mBAAmB;AAC1G,SAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;AACtF,QAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;AACxD,wBAAsB,SAAS,IAAI;AACnC,SAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;AACrE,QAAM,WAAW,KAAK;AACtB,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,gCAAgC;AAC/F,SAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,EACtC,WAAW,cAAc,QAAQ,EACjC,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,EAClE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;AACzE,QAAM,aAAuB,CAAC;AAC9B,aAAW,KAAK,QAAQ,SAAS,+BAA+B,EAAG,YAAW,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;AAC9F,aAAW,KAAK,QAAQ,SAAS,eAAe,EAAG,YAAW,KAAK,EAAE,CAAC,CAAC;AACvE,aAAW,KAAK,QAAQ,KAAK,CAAC;AAC9B,aAAW,aAAa,YAAY;AAClC,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,SAAS;AAAA,IAC/B,QAAQ;AACN;AAAA,IACF;AACA,UAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;AACjF,UAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;AAClF,QAAI,CAAC,OAAO,SAAS,CAAC,EAAG;AACzB,QAAI,IAAI,EAAG,OAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,CAAC,EAAE;AAChG,UAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;AAClD,QAAI,QAAQ,EAAG,OAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,CAAC,EAAE;AAChH,WAAO,EAAE,OAAO,KAAK,OAAO;AAAA,EAC9B;AACA,QAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;AAC5F,MAAI,CAAC,KAAK,kBAAmB,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,8BAA8B;AAC3G,QAAM,SAAS,aAAa;AAC5B,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU;AAAA,QACR,EAAE,MAAM,UAAU,SAAS,KAAK,kBAAkB;AAAA,QAClD,EAAE,MAAM,QAAQ,SAAS,aAAa,MAAM,QAAQ,EAAE;AAAA,MACxD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7G,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,SAAU,OAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAC5I,QAAM,EAAE,OAAO,IAAI,IAAI,gBAAgB,OAAO;AAC9C,SAAO;AAAA,IACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,GAAG;AAAA,IAC5E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,YAAY,OAAO,OAAO,IAAI,CAAC;AAAA,EAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;AACzE,QAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;AACzE,QAAM,WAAW,cAAc,QAAQ;AACvC,QAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;AACnE,SAAO;AAAA,IACL,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,SAAS,CAAC;AAAA,EAC3E;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,8BAA8B;AAE/D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,eAAe,UAAU,cAAc;AAC7C;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,mBAAa;AACb,YAAM,eAAe,MAAM,WAAW;AACtC,YAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,sEAAsE;AACjG,aAAO,kBAAkB,MAAM,IAAI;AAAA,IACrC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,mBAAmB,KAAK,mBAAmB;AAAA,IACzD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,aAAc,QAAO,aAAa,MAAM,QAAQ;AACpD,UAAI,SAAS,KAAK,EAAE,WAAW,GAAG;AAChC,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,QAAQ,eAAe,CAAC;AAAA,QACnG;AAAA,MACF;AACA,aAAO,iBAAiB,MAAM,QAAQ;AAAA,IACxC;AAAA,EACF;AACF;","names":[]}