@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/chunk-PPYSEKFM.js

@@ -0,0 +1,182 @@
+// src/benchmarks/humaneval.ts
+import { execFile } from "child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { gunzipSync } from "zlib";
+var humanevalUrl = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz";
+var dockerImage = "python:3.12-slim";
+var dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 2e4);
+async function loadHumanEval(limit, offset = 0) {
+  const localGz = process.env.HUMANEVAL_GZ;
+  let gz;
+  if (localGz) {
+    gz = readFileSync(localGz);
+  } else {
+    const res = await fetch(humanevalUrl);
+    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`);
+    gz = Buffer.from(await res.arrayBuffer());
+  }
+  const text = gunzipSync(gz).toString("utf8");
+  const tasks = [];
+  for (const line of text.split("\n")) {
+    if (line.trim() === "") continue;
+    const d = JSON.parse(line);
+    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {
+      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`);
+    }
+    tasks.push({
+      taskId: d.task_id,
+      prompt: d.prompt,
+      test: d.test,
+      entryPoint: d.entry_point,
+      ...d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}
+    });
+  }
+  if (tasks.length === 0) throw new Error("HumanEval parsed to 0 tasks");
+  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`);
+  return tasks.slice(offset, offset + limit);
+}
+var solveInstruction = "Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.";
+function basePrompt(task) {
+  return `${solveInstruction}
+
+\`\`\`python
+${task.prompt}\`\`\``;
+}
+function extractCode(reply) {
+  const fenced = reply.match(/```(?:python|py)?\s*\n([\s\S]*?)```/i);
+  if (fenced && typeof fenced[1] === "string") return fenced[1].trim();
+  return reply.trim();
+}
+function buildProgram(task, candidate) {
+  return `${task.prompt}
+${candidate}
+
+${task.test}
+
+check(${task.entryPoint})
+`;
+}
+var dockerRunSeq = 0;
+function runChecker(task, candidate) {
+  const dir = mkdtempSync(join(tmpdir(), "hev-"));
+  writeFileSync(join(dir, "p.py"), buildProgram(task, candidate));
+  const name = `hev-${process.pid}-${dockerRunSeq++}`;
+  return new Promise((resolvePromise, reject) => {
+    let settled = false;
+    const cleanup = () => {
+      rmSync(dir, { recursive: true, force: true });
+      execFile("docker", ["rm", "-f", name], () => {
+      });
+    };
+    const finish = (res) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(backstop);
+      cleanup();
+      resolvePromise(res);
+    };
+    const fail = (e) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(backstop);
+      cleanup();
+      reject(e);
+    };
+    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3e3);
+    execFile(
+      "docker",
+      [
+        "run",
+        "--rm",
+        "--name",
+        name,
+        "--network=none",
+        "--cpus=1",
+        "--memory=512m",
+        "-v",
+        `${dir}:/w:ro`,
+        "-w",
+        "/w",
+        dockerImage,
+        "python",
+        "/w/p.py"
+      ],
+      { timeout: dockerTimeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024 },
+      (err, _stdout, stderr) => {
+        if (err) {
+          const e = err;
+          if (e.code === "ENOENT") {
+            fail(new Error("docker binary not found on PATH \u2014 cannot run the deployable checker"));
+            return;
+          }
+          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
+            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
+            return;
+          }
+          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {
+            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
+            return;
+          }
+          finish({ pass: 0, detail: (stderr || "").slice(-600) || "timed out (no output)" });
+          return;
+        }
+        finish({ pass: 1 });
+      }
+    );
+  });
+}
+function toBenchTask(t) {
+  const metadata = {
+    promptHeader: t.prompt,
+    test: t.test,
+    entryPoint: t.entryPoint,
+    ...t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}
+  };
+  return { id: t.taskId, prompt: basePrompt(t), metadata };
+}
+function taskFromMeta(task) {
+  const m = task.metadata;
+  if (!m?.promptHeader || !m.test || !m.entryPoint) {
+    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`);
+  }
+  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint };
+}
+function createHumanEvalAdapter() {
+  return {
+    name: "humaneval",
+    async preflight() {
+      await new Promise((resolve, reject) => {
+        execFile("docker", ["version", "--format", "{{.Server.Version}}"], (err) => {
+          if (err) reject(new Error("HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)"));
+          else resolve();
+        });
+      });
+    },
+    async loadTasks(opts) {
+      const offset = Number(process.env.OFFSET ?? 0);
+      const all = await loadHumanEval(opts?.ids ? 164 : opts?.limit ?? 8, offset);
+      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all;
+      return picked.map(toBenchTask);
+    },
+    async judge(task, artifact) {
+      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact));
+      return { resolved: pass === 1, score: pass, detail: pass === 1 ? "tests passed" : "tests failed" };
+    },
+    async goldArtifact(task) {
+      const m = task.metadata;
+      const sol = m?.canonicalSolution;
+      return sol ? `${m.promptHeader}${sol}` : void 0;
+    }
+  };
+}
+
+export {
+  loadHumanEval,
+  basePrompt,
+  extractCode,
+  runChecker,
+  createHumanEvalAdapter
+};
+//# sourceMappingURL=chunk-PPYSEKFM.js.map

package/dist/chunk-PPYSEKFM.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/humaneval.ts"],"sourcesContent":["/**\n * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the\n * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout\n * through the `Supervisor` that self-corrects across rounds, vs blind\n * random@k. This is the experiment `humaneval-gate.mts` names as \"the next one\" —\n * the gate measures SELECTION over stateless single completions; this measures\n * whether observe→steer (self-correction) beats blind compute at equal k.\n *\n * Worker artifact = the model's reply (a Python function, fenced or raw). The\n * DETERMINISTIC judge runs the candidate against the task's own `test` in an\n * isolated `--network=none` python container — exit 0 = pass. No gold\n * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only\n * to self-verify the judge before spending tokens.\n *\n * The primitives (loader / extractor / Docker checker) live here and are reused by\n * `humaneval-gate.mts` — one home, no duplication.\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { gunzipSync } from 'node:zlib'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'\nconst dockerImage = 'python:3.12-slim'\nconst dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)\n\nexport interface HumanEvalTask {\n  taskId: string\n  prompt: string\n  test: string\n  entryPoint: string\n  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */\n  canonicalSolution?: string\n}\n\n/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch\n *  or a malformed line — a silently-short task set would poison the gate. `offset`\n *  selects a deeper slice (the later tasks are harder) so the worker has a\n *  correctable middle band rather than a saturated easy prefix. */\nexport async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {\n  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits\n  // (429) under repeated runs. Fall back to the network fetch when unset.\n  const localGz = process.env.HUMANEVAL_GZ\n  let gz: Buffer\n  if (localGz) {\n    gz = readFileSync(localGz)\n  } else {\n    const res = await fetch(humanevalUrl)\n    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)\n    gz = Buffer.from(await res.arrayBuffer())\n  }\n  const text = gunzipSync(gz).toString('utf8')\n  const tasks: HumanEvalTask[] = []\n  for (const line of text.split('\\n')) {\n    if (line.trim() === '') continue\n    const d = JSON.parse(line) as {\n      task_id?: string\n      prompt?: string\n      test?: string\n      entry_point?: string\n      canonical_solution?: string\n    }\n    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {\n      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)\n    }\n    tasks.push({\n      taskId: d.task_id,\n      prompt: d.prompt,\n      test: d.test,\n      entryPoint: d.entry_point,\n      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),\n    })\n  }\n  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')\n  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)\n  return tasks.slice(offset, offset + limit)\n}\n\nconst solveInstruction =\n  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'\n\nexport function basePrompt(task: HumanEvalTask): string {\n  return `${solveInstruction}\\n\\n\\`\\`\\`python\\n${task.prompt}\\`\\`\\``\n}\n\n/** Extract the function source from a model reply: prefer a fenced ```python (or\n *  bare ```) block, else fall back to the raw text. The deployable program adds the\n *  prompt header (imports + signature context), so a candidate that returns only a\n *  body still runs; a candidate that re-defines the function shadows the header. */\nexport function extractCode(reply: string): string {\n  const fenced = reply.match(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i)\n  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()\n  return reply.trim()\n}\n\n/** The deployable test program: the prompt header (imports + signature/docstring the\n *  model was given), then the candidate (its def shadows the header's stub), then the\n *  task's own check() suite and the call. No gold solution anywhere. */\nfunction buildProgram(task: HumanEvalTask, candidate: string): string {\n  return `${task.prompt}\\n${candidate}\\n\\n${task.test}\\n\\ncheck(${task.entryPoint})\\n`\n}\n\nexport interface CheckResult {\n  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */\n  pass: number\n  /** On failure: the interpreter stderr tail (traceback / failing assertion). The\n   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */\n  detail?: string\n}\n\n/** Run one candidate's deployable test program in an isolated container:\n *  `docker run --rm --network=none -v <tmp>:/w -w /w <img> python /w/p.py`.\n *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image\n *  pull failure) is NOT a test failure — it throws so the harness fails loud rather\n *  than scoring every candidate 0 from a broken checker. */\nlet dockerRunSeq = 0\n\nexport function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {\n  const dir = mkdtempSync(join(tmpdir(), 'hev-'))\n  writeFileSync(join(dir, 'p.py'), buildProgram(task, candidate))\n  // Unique container name so we can force-reap it regardless of the docker client's state.\n  const name = `hev-${process.pid}-${dockerRunSeq++}`\n  return new Promise<CheckResult>((resolvePromise, reject) => {\n    let settled = false\n    const cleanup = () => {\n      rmSync(dir, { recursive: true, force: true })\n      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung\n      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;\n      // the name is unique, so no reuse race).\n      execFile('docker', ['rm', '-f', name], () => {})\n    }\n    const finish = (res: CheckResult) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      resolvePromise(res)\n    }\n    const fail = (e: Error) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      reject(e)\n    }\n    // A hung container can leave the docker client stuck forwarding SIGTERM, so the\n    // execFile callback never fires. This guarantees resolution (and reap) after the\n    // timeout, independent of the callback.\n    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3000)\n    execFile(\n      'docker',\n      [\n        'run',\n        '--rm',\n        '--name',\n        name,\n        '--network=none',\n        '--cpus=1',\n        '--memory=512m',\n        '-v',\n        `${dir}:/w:ro`,\n        '-w',\n        '/w',\n        dockerImage,\n        'python',\n        '/w/p.py',\n      ],\n      { timeout: dockerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },\n      (err, _stdout, stderr) => {\n        if (err) {\n          const e = err as NodeJS.ErrnoException & { killed?: boolean; code?: number | string }\n          if (e.code === 'ENOENT') {\n            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))\n            return\n          }\n          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {\n            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))\n            return\n          }\n          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {\n            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))\n            return\n          }\n          // killed-by-timeout or a non-zero exit (assert failure / error) are genuine\n          // test FAILURES — score 0, do not throw. Carry the stderr tail as the\n          // execution-grounded failure detail (empty ⇒ timeout/SIGKILL left no output).\n          finish({ pass: 0, detail: (stderr || '').slice(-600) || 'timed out (no output)' })\n          return\n        }\n        finish({ pass: 1 })\n      },\n    )\n  })\n}\n\n/** A HumanEval task carries its checker inputs in metadata so the deterministic\n *  judge can rebuild the deployable program from a `BenchTask` alone. */\ninterface HumanEvalMeta extends Record<string, unknown> {\n  promptHeader: string\n  test: string\n  entryPoint: string\n  canonicalSolution?: string\n}\n\nfunction toBenchTask(t: HumanEvalTask): BenchTask {\n  const metadata: HumanEvalMeta = {\n    promptHeader: t.prompt,\n    test: t.test,\n    entryPoint: t.entryPoint,\n    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),\n  }\n  return { id: t.taskId, prompt: basePrompt(t), metadata }\n}\n\nfunction taskFromMeta(task: BenchTask): HumanEvalTask {\n  const m = task.metadata as HumanEvalMeta | undefined\n  if (!m?.promptHeader || !m.test || !m.entryPoint) {\n    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)\n  }\n  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }\n}\n\n/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle\n *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */\nexport function createHumanEvalAdapter(): BenchmarkAdapter {\n  return {\n    name: 'humaneval',\n    async preflight() {\n      // The judge is the only hard dependency; it fails loud on a missing/broken\n      // docker, so a cheap presence check here gives an earlier, clearer signal.\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {\n          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))\n          else resolve()\n        })\n      })\n    },\n    async loadTasks(opts?: LoadOptions) {\n      const offset = Number(process.env.OFFSET ?? 0)\n      // Pull a generous window when filtering by id, else exactly `limit` from offset.\n      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)\n      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all\n      return picked.map(toBenchTask)\n    },\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))\n      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }\n    },\n    async goldArtifact(task: BenchTask) {\n      const m = task.metadata as HumanEvalMeta | undefined\n      const sol = m?.canonicalSolution\n      // Return the COMPLETE function (signature header + canonical body), i.e. what a\n      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose\n      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its\n      // leading indent and breaks it, so a body-only gold fails its own judge. A full\n      // def starts at column 0, trims safely, and self-verifies.\n      return sol ? `${m!.promptHeader}${sol}` : undefined\n    },\n  }\n}\n"],"mappings":";AAkBA,SAAS,gBAAgB;AACzB,SAAS,aAAa,cAAc,QAAQ,qBAAqB;AACjE,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAG3B,IAAM,eAAe;AACrB,IAAM,cAAc;AACpB,IAAM,kBAAkB,OAAO,QAAQ,IAAI,qBAAqB,GAAK;AAerE,eAAsB,cAAc,OAAe,SAAS,GAA6B;AAGvF,QAAM,UAAU,QAAQ,IAAI;AAC5B,MAAI;AACJ,MAAI,SAAS;AACX,SAAK,aAAa,OAAO;AAAA,EAC3B,OAAO;AACL,UAAM,MAAM,MAAM,MAAM,YAAY;AACpC,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,KAAK,YAAY,EAAE;AAClF,SAAK,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC;AAAA,EAC1C;AACA,QAAM,OAAO,WAAW,EAAE,EAAE,SAAS,MAAM;AAC3C,QAAM,QAAyB,CAAC;AAChC,aAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,QAAI,KAAK,KAAK,MAAM,GAAI;AACxB,UAAM,IAAI,KAAK,MAAM,IAAI;AAOzB,QAAI,CAAC,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,aAAa;AACxD,YAAM,IAAI,MAAM,+BAA+B,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrE;AACA,UAAM,KAAK;AAAA,MACT,QAAQ,EAAE;AAAA,MACV,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,YAAY,EAAE;AAAA,MACd,GAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,IAAI,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B;AACrE,MAAI,UAAU,MAAM,OAAQ,OAAM,IAAI,MAAM,UAAU,MAAM,oBAAoB,MAAM,MAAM,EAAE;AAC9F,SAAO,MAAM,MAAM,QAAQ,SAAS,KAAK;AAC3C;AAEA,IAAM,mBACJ;AAEK,SAAS,WAAW,MAA6B;AACtD,SAAO,GAAG,gBAAgB;AAAA;AAAA;AAAA,EAAqB,KAAK,MAAM;AAC5D;AAMO,SAAS,YAAY,OAAuB;AACjD,QAAM,SAAS,MAAM,MAAM,sCAAsC;AACjE,MAAI,UAAU,OAAO,OAAO,CAAC,MAAM,SAAU,QAAO,OAAO,CAAC,EAAE,KAAK;AACnE,SAAO,MAAM,KAAK;AACpB;AAKA,SAAS,aAAa,MAAqB,WAA2B;AACpE,SAAO,GAAG,KAAK,MAAM;AAAA,EAAK,SAAS;AAAA;AAAA,EAAO,KAAK,IAAI;AAAA;AAAA,QAAa,KAAK,UAAU;AAAA;AACjF;AAeA,IAAI,eAAe;AAEZ,SAAS,WAAW,MAAqB,WAAyC;AACvF,QAAM,MAAM,YAAY,KAAK,OAAO,GAAG,MAAM,CAAC;AAC9C,gBAAc,KAAK,KAAK,MAAM,GAAG,aAAa,MAAM,SAAS,CAAC;AAE9D,QAAM,OAAO,OAAO,QAAQ,GAAG,IAAI,cAAc;AACjD,SAAO,IAAI,QAAqB,CAAC,gBAAgB,WAAW;AAC1D,QAAI,UAAU;AACd,UAAM,UAAU,MAAM;AACpB,aAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAI5C,eAAS,UAAU,CAAC,MAAM,MAAM,IAAI,GAAG,MAAM;AAAA,MAAC,CAAC;AAAA,IACjD;AACA,UAAM,SAAS,CAAC,QAAqB;AACnC,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,qBAAe,GAAG;AAAA,IACpB;AACA,UAAM,OAAO,CAAC,MAAa;AACzB,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,aAAO,CAAC;AAAA,IACV;AAIA,UAAM,WAAW,WAAW,MAAM,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,kBAAkB,GAAI;AAC7E;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,GAAG;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,EAAE,SAAS,iBAAiB,YAAY,WAAW,WAAW,IAAI,OAAO,KAAK;AAAA,MAC9E,CAAC,KAAK,SAAS,WAAW;AACxB,YAAI,KAAK;AACP,gBAAM,IAAI;AACV,cAAI,EAAE,SAAS,UAAU;AACvB,iBAAK,IAAI,MAAM,0EAAqE,CAAC;AACrF;AAAA,UACF;AACA,cAAI,8GAA8G,KAAK,MAAM,GAAG;AAC9H,iBAAK,IAAI,MAAM,8BAA8B,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AACpE;AAAA,UACF;AACA,cAAI,qHAAqH,KAAK,MAAM,GAAG;AACrI,iBAAK,IAAI,MAAM,gBAAgB,WAAW,iBAAiB,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AAClF;AAAA,UACF;AAIA,iBAAO,EAAE,MAAM,GAAG,SAAS,UAAU,IAAI,MAAM,IAAI,KAAK,wBAAwB,CAAC;AACjF;AAAA,QACF;AACA,eAAO,EAAE,MAAM,EAAE,CAAC;AAAA,MACpB;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAWA,SAAS,YAAY,GAA6B;AAChD,QAAM,WAA0B;AAAA,IAC9B,cAAc,EAAE;AAAA,IAChB,MAAM,EAAE;AAAA,IACR,YAAY,EAAE;AAAA,IACd,GAAI,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC;AAAA,EAC1E;AACA,SAAO,EAAE,IAAI,EAAE,QAAQ,QAAQ,WAAW,CAAC,GAAG,SAAS;AACzD;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,IAAI,KAAK;AACf,MAAI,CAAC,GAAG,gBAAgB,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY;AAChD,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,2BAA2B;AAAA,EAC7E;AACA,SAAO,EAAE,QAAQ,KAAK,IAAI,QAAQ,EAAE,cAAc,MAAM,EAAE,MAAM,YAAY,EAAE,WAAW;AAC3F;AAIO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,YAAY;AAGhB,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,WAAW,YAAY,qBAAqB,GAAG,CAAC,QAAQ;AAC1E,cAAI,IAAK,QAAO,IAAI,MAAM,kFAAkF,CAAC;AAAA,cACxG,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,IACA,MAAM,UAAU,MAAoB;AAClC,YAAM,SAAS,OAAO,QAAQ,IAAI,UAAU,CAAC;AAE7C,YAAM,MAAM,MAAM,cAAc,MAAM,MAAM,MAAO,MAAM,SAAS,GAAI,MAAM;AAC5E,YAAM,SAAS,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,SAAS,EAAE,MAAM,CAAC,IAAI;AAC7E,aAAO,OAAO,IAAI,WAAW;AAAA,IAC/B;AAAA,IACA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,KAAK,IAAI,MAAM,WAAW,aAAa,IAAI,GAAG,YAAY,QAAQ,CAAC;AAC3E,aAAO,EAAE,UAAU,SAAS,GAAG,OAAO,MAAM,QAAQ,SAAS,IAAI,iBAAiB,eAAe;AAAA,IACnG;AAAA,IACA,MAAM,aAAa,MAAiB;AAClC,YAAM,IAAI,KAAK;AACf,YAAM,MAAM,GAAG;AAMf,aAAO,MAAM,GAAG,EAAG,YAAY,GAAG,GAAG,KAAK;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-PUIRNYI7.js

@@ -0,0 +1,189 @@
+// src/benchmarks/cadbench.ts
+import { readFile as readFile2 } from "fs/promises";
+
+// src/worker-blender.ts
+import { execFile } from "child_process";
+import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+import { routerChatWithUsage } from "@tangle-network/agent-runtime/loops";
+var execFileAsync = promisify(execFile);
+async function runLocal(cmd, args, cwd, timeoutMs = 18e4) {
+  try {
+    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs });
+    return { code: 0, stdout, stderr };
+  } catch (err) {
+    const e = err;
+    return { code: typeof e.code === "number" ? e.code : 1, stdout: e.stdout ?? "", stderr: e.stderr ?? e.message ?? String(err) };
+  }
+}
+var RUNNER_PY = `
+import bpy, sys, math, mathutils, traceback, os
+agent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])
+bpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()
+ok=True
+try:
+    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}
+    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)
+except Exception as e:
+    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False
+meshes=[o for o in bpy.context.scene.objects if o.type=='MESH']
+if not meshes:
+    print('NO_MESH'); sys.exit(0 if ok else 3)
+mn=[1e18]*3; mx=[-1e18]*3
+for o in meshes:
+    for c in o.bound_box:
+        w=o.matrix_world @ mathutils.Vector(c)
+        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])
+center=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))
+size=max(mx[i]-mn[i] for i in range(3)) or 1.0
+# standardize: drop any agent-added cameras/lights
+for o in list(bpy.context.scene.objects):
+    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)
+w=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w
+w.use_nodes=True
+try: w.node_tree.nodes['Background'].inputs[1].default_value=0.6
+except Exception: pass
+bpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))
+bpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam
+sc=bpy.context.scene
+sc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False
+sc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False
+dist=size*2.4
+el=math.radians(58)
+for v in range(nviews):
+    az=math.radians(40 + v*360.0/nviews)
+    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist
+    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()
+    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)
+print('RENDER_DONE')
+`.trim();
+async function renderBpy(script, opts = {}) {
+  const views = Math.max(1, opts.views ?? 4);
+  const dir = await mkdtemp(join(tmpdir(), "blender-judge-"));
+  const runnerPath = join(dir, "runner.py");
+  const scriptPath = join(dir, "model.py");
+  try {
+    await writeFile(runnerPath, RUNNER_PY);
+    await writeFile(scriptPath, script);
+    const run = await runLocal("xvfb-run", ["-a", "blender", "--background", "--python", runnerPath, "--", scriptPath, dir, String(views)], dir);
+    const out = `${run.stdout}
+${run.stderr}`;
+    const built = /RENDER_DONE/.test(out);
+    if (!built) return { built: false, renders: [], error: /(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400) };
+    const renders = [];
+    for (let v = 0; v < views; v++) {
+      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => void 0);
+      if (buf) renders.push(`data:image/png;base64,${buf.toString("base64")}`);
+    }
+    return { built: renders.length > 0, renders };
+  } finally {
+    await rm(dir, { recursive: true, force: true }).catch(() => {
+    });
+  }
+}
+
+// src/benchmarks/cadbench.ts
+function must(name) {
+  const v = process.env[name];
+  if (!v) throw new Error(`env ${name} is required for the CADBench judge`);
+  return v;
+}
+async function judgeCriteria(instruction, script, criteria, renders) {
+  const base = (process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1").replace(/\/$/, "");
+  const key = must("TANGLE_API_KEY");
+  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join("\n");
+  const text = `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:
+"${instruction}"
+
+Below are ${renders.length} rendered views of the produced model, and the script that built it. For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.
+
+CRITERIA:
+${numbered}
+
+SCRIPT:
+\`\`\`python
+${script.slice(0, 6e3)}
+\`\`\``;
+  const content = [{ type: "text", text }];
+  for (const url of renders) content.push({ type: "image_url", image_url: { url } });
+  const res = await fetch(`${base}/chat/completions`, {
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
+    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: "user", content }] })
+  });
+  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`);
+  const data = await res.json();
+  const raw = data.choices?.[0]?.message?.content ?? "";
+  const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw);
+  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` };
+  let arr;
+  try {
+    arr = JSON.parse(m[0].toLowerCase());
+  } catch {
+    return { passed: criteria.map(() => false), note: "judge verdict not valid JSON" };
+  }
+  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : [];
+  const passed = criteria.map((_, i) => bools[i] === true);
+  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` };
+}
+function createCadBenchAdapter() {
+  let cache = null;
+  async function load() {
+    if (cache) return cache;
+    const path = process.env.CADBENCH_PATH;
+    if (!path) throw new Error("CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)");
+    const text = await readFile2(path, "utf8");
+    cache = text.split("\n").filter((l) => l.trim()).map((l) => {
+      const r = JSON.parse(l);
+      return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? "", type: r.type ?? "", criteria: r.criteria } };
+    });
+    return cache;
+  }
+  return {
+    name: "cadbench",
+    async preflight() {
+      const { execFile: execFile2 } = await import("child_process");
+      const { promisify: promisify2 } = await import("util");
+      const exec = promisify2(execFile2);
+      try {
+        await exec("xvfb-run", ["-a", "blender", "--version"], { timeout: 3e4 });
+      } catch (err) {
+        throw new Error(
+          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}
+Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \`xvfb-run -a blender --background --python\`.`
+        );
+      }
+      await load();
+    },
+    async loadTasks(opts = {}) {
+      let rows = await load();
+      if (opts.ids) rows = rows.filter((r) => opts.ids.includes(r.id));
+      const t = process.env.CADBENCH_TYPE;
+      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase());
+      if (opts.limit != null) rows = rows.slice(0, opts.limit);
+      return rows.map((r) => ({ id: r.id, prompt: r.instruction, metadata: r.meta }));
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const meta = task.metadata;
+      const criteria = meta.criteria ?? [];
+      if (!artifact.trim()) return { resolved: false, score: 0, detail: "empty artifact" };
+      if (criteria.length === 0) return { resolved: false, score: 0, detail: "task has no criteria" };
+      const r = await renderBpy(artifact, { views: 4 });
+      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? "no mesh"}` };
+      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders);
+      const score = passed.filter(Boolean).length / criteria.length;
+      return { resolved: score === 1, score, detail: note };
+    }
+  };
+}
+
+export {
+  createCadBenchAdapter
+};
+//# sourceMappingURL=chunk-PUIRNYI7.js.map

package/dist/chunk-PUIRNYI7.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/cadbench.ts","../src/worker-blender.ts"],"sourcesContent":["/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const res = await fetch(`${base}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n  })\n  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n  const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const raw = data.choices?.[0]?.message?.content ?? ''\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n","/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const { content, usage: u } = await routerChatWithUsage(\n        cfg,\n        [\n          { role: 'system', content: directive },\n          { role: 'user', content: user },\n        ],\n        { temperature: 0.3 },\n      )\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n"],"mappings":";AAaA,SAAS,YAAAA,iBAAgB;;;ACFzB,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,IAAI,iBAAiB;AACjD,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAK1B,SAAS,2BAA2B;AAIpC,IAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;AAChJ,MAAI;AACF,UAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,KAAK,MAAM,EAAE,KAAK,WAAW,KAAK,IAAI,SAAS,UAAU,CAAC;AACzG,WAAO,EAAE,MAAM,GAAG,QAAQ,OAAO;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,WAAO,EAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO,GAAG,QAAQ,EAAE,UAAU,IAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,EAAE;AAAA,EAC/H;AACF;AAcA,IAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAwChB,KAAK;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;AAC7I,QAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;AACzC,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;AAC1D,QAAM,aAAa,KAAK,KAAK,WAAW;AACxC,QAAM,aAAa,KAAK,KAAK,UAAU;AACvC,MAAI;AACF,UAAM,UAAU,YAAY,SAAS;AACrC,UAAM,UAAU,YAAY,MAAM;AAClC,UAAM,MAAM,MAAM,SAAS,YAAY,CAAC,MAAM,WAAW,gBAAgB,YAAY,YAAY,MAAM,YAAY,KAAK,OAAO,KAAK,CAAC,GAAG,GAAG;AAC3I,UAAM,MAAM,GAAG,IAAI,MAAM;AAAA,EAAK,IAAI,MAAM;AACxC,UAAM,QAAQ,cAAc,KAAK,GAAG;AACpC,QAAI,CAAC,MAAO,QAAO,EAAE,OAAO,OAAO,SAAS,CAAC,GAAG,OAAQ,kCAAkC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,MAAM,IAAI,EAAG;AACpI,UAAM,UAAoB,CAAC;AAC3B,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,MAAS;AAC5E,UAAI,IAAK,SAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,CAAC,EAAE;AAAA,IACzE;AACA,WAAO,EAAE,OAAO,QAAQ,SAAS,GAAG,QAAQ;AAAA,EAC9C,UAAE;AACA,UAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAChE;AACF;;;AD1FA,SAAS,KAAK,MAAsB;AAClC,QAAM,IAAI,QAAQ,IAAI,IAAI;AAC1B,MAAI,CAAC,EAAG,OAAM,IAAI,MAAM,OAAO,IAAI,qCAAqC;AACxE,SAAO;AACT;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;AAC9C,QAAM,QAAQ,QAAQ,IAAI,eAAe,kCAAkC,QAAQ,OAAO,EAAE;AAC5F,QAAM,MAAM,KAAK,gBAAgB;AACjC,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,WAAW,SAAS,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,IAAI;AACnE,QAAM,OACJ;AAAA,GAAsG,WAAW;AAAA;AAAA,YACpG,QAAQ,MAAM,8TAEY,SAAS,MAAM;AAAA;AAAA;AAAA,EAA4E,QAAQ;AAAA;AAAA;AAAA;AAAA,EAA8B,OAAO,MAAM,GAAG,GAAI,CAAC;AAAA;AAC/L,QAAM,UAAqB,CAAC,EAAE,MAAM,QAAQ,KAAK,CAAC;AAClD,aAAW,OAAO,QAAS,SAAQ,KAAK,EAAE,MAAM,aAAa,WAAW,EAAE,IAAI,EAAE,CAAC;AACjF,QAAM,MAAM,MAAM,MAAM,GAAG,IAAI,qBAAqB;AAAA,IAClD,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,GAAG,GAAG;AAAA,IAC9E,MAAM,KAAK,UAAU,EAAE,OAAO,YAAY,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,MAAM,QAAQ,QAAQ,CAAC,EAAE,CAAC;AAAA,EACzG,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,SAAS,KAAK,IAAI,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAChG,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,MAAM,KAAK,UAAU,CAAC,GAAG,SAAS,WAAW;AACnD,QAAM,IAAI,iCAAiC,KAAK,GAAG;AACnD,MAAI,CAAC,EAAG,QAAO,EAAE,QAAQ,SAAS,IAAI,MAAM,KAAK,GAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE,CAAC,GAAG;AACrH,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,MAAM,EAAE,CAAC,EAAE,YAAY,CAAC;AAAA,EACrC,QAAQ;AACN,WAAO,EAAE,QAAQ,SAAS,IAAI,MAAM,KAAK,GAAG,MAAM,+BAA+B;AAAA,EACnF;AACA,QAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,MAAM,IAAI,IAAI,CAAC;AAEjE,QAAM,SAAS,SAAS,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,MAAM,IAAI;AACvD,SAAO,EAAE,QAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,EAAE,MAAM,IAAI,SAAS,MAAM,YAAY;AACxF;AAEO,SAAS,wBAA0C;AACxD,MAAI,QAA+E;AAEnF,iBAAe,OAAuC;AACpD,QAAI,MAAO,QAAO;AAClB,UAAM,OAAO,QAAQ,IAAI;AACzB,QAAI,CAAC,KAAM,OAAM,IAAI,MAAM,qGAAqG;AAChI,UAAM,OAAO,MAAMC,UAAS,MAAM,MAAM;AACxC,YAAQ,KACL,MAAM,IAAI,EACV,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EACtB,IAAI,CAAC,MAAM;AACV,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,aAAO,EAAE,IAAI,EAAE,IAAI,aAAa,EAAE,aAAa,MAAM,EAAE,MAAM,EAAE,QAAQ,IAAI,MAAM,EAAE,QAAQ,IAAI,UAAU,EAAE,SAAS,EAAE;AAAA,IACxH,CAAC;AACH,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,EAAE,UAAAC,UAAS,IAAI,MAAM,OAAO,eAAoB;AACtD,YAAM,EAAE,WAAAC,WAAU,IAAI,MAAM,OAAO,MAAW;AAC9C,YAAM,OAAOA,WAAUD,SAAQ;AAC/B,UAAI;AACF,cAAM,KAAK,YAAY,CAAC,MAAM,WAAW,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MAC5E,SAAS,KAAK;AACZ,cAAM,IAAI;AAAA,UACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAAG,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA,QAEhG;AAAA,MACF;AACA,YAAM,KAAK;AAAA,IACb;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,OAAO,MAAM,KAAK;AACtB,UAAI,KAAK,IAAK,QAAO,KAAK,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAEhE,YAAM,IAAI,QAAQ,IAAI;AACtB,UAAI,EAAG,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;AAC9E,UAAI,KAAK,SAAS,KAAM,QAAO,KAAK,MAAM,GAAG,KAAK,KAAK;AACvD,aAAO,KAAK,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,aAAa,UAAU,EAAE,KAA2C,EAAE;AAAA,IACjI;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,KAAK;AAClB,YAAM,WAAW,KAAK,YAAY,CAAC;AACnC,UAAI,CAAC,SAAS,KAAK,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AACnF,UAAI,SAAS,WAAW,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,uBAAuB;AAC9F,YAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;AAChD,UAAI,CAAC,EAAE,MAAO,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,yBAAyB,EAAE,SAAS,SAAS,GAAG;AAC1G,YAAM,EAAE,QAAQ,KAAK,IAAI,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;AACvF,YAAM,QAAQ,OAAO,OAAO,OAAO,EAAE,SAAS,SAAS;AACvD,aAAO,EAAE,UAAU,UAAU,GAAG,OAAO,QAAQ,KAAK;AAAA,IACtD;AAAA,EACF;AACF;","names":["readFile","readFile","execFile","promisify"]}

package/dist/chunk-R36V2VP7.js

@@ -0,0 +1,169 @@
+import {
+  runVenvPython
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/tau-bench-shared.ts
+import { readFile, stat } from "fs/promises";
+import { resolve } from "path";
+var tauResultsOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:text|path|json)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+async function assertPath(path, label, benchName) {
+  try {
+    await stat(path);
+  } catch (err) {
+    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+  }
+}
+function benchDir(config) {
+  return process.env[config.dirEnv];
+}
+function benchDomain(config) {
+  return process.env[config.domainEnv] ?? config.defaultDomain;
+}
+function rowToTask(row, config, split) {
+  const meta = {
+    taskId: row.id,
+    domain: row.domain,
+    split,
+    userScenario: row.user_scenario,
+    description: row.description,
+    evaluationCriteria: row.evaluation_criteria
+  };
+  return {
+    id: row.id,
+    split,
+    prompt: [
+      config.taskIntro,
+      `Run this task in the official ${row.domain} domain.`,
+      "The benchmark is a simulated multi-turn user/tool conversation.",
+      "",
+      typeof row.user_scenario === "string" ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),
+      "",
+      "Return the path to the official tau results.json or trajectory file containing this task run."
+    ].join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task, benchName) {
+  const md = task.metadata;
+  if (!md || typeof md.taskId !== "string" || typeof md.domain !== "string") {
+    throw new Error(`${benchName} task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts, config, split) {
+  let tasks = rows.map((row) => rowToTask(row, config, split));
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+async function loadFixtures(config, opts) {
+  const rows = JSON.parse(await readFile(config.fixturePath, "utf8"));
+  console.warn(`[${config.name}] ${config.fixturesEnv}=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectRows(rows, opts, config, opts.split);
+}
+async function loadOfficialTasks(config, root, opts) {
+  const domain = benchDomain(config);
+  const script = `
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+domain = sys.argv[2]
+split = sys.argv[3] or None
+sys.path.insert(0, str(root / "src"))
+from tau2.registry import registry
+loader = registry.get_tasks_loader(domain)
+tasks = loader(split)
+rows = []
+for task in tasks:
+    row = task.model_dump(mode="json")
+    row["domain"] = domain
+    rows.append(row)
+print(json.dumps(rows))
+`;
+  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ""]);
+  return selectRows(JSON.parse(stdout), opts, config, opts.split);
+}
+async function scoreOfficialTrajectory(root, meta, artifactPath) {
+  const script = `
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+artifact = Path(sys.argv[3])
+sys.path.insert(0, str(root / "src"))
+from tau2.data_model.simulation import Results
+from tau2.scripts.evaluate_trajectories import compute_simulation_rewards
+results = Results.load(artifact)
+updated = compute_simulation_rewards(results)
+scores = []
+for sim in updated.simulations:
+    if sim.task_id == task_id and sim.reward_info is not None:
+        scores.append(float(sim.reward_info.reward))
+if not scores:
+    raise SystemExit(f"no scored simulations for task_id={task_id} in {artifact}")
+print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores}))
+`;
+  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0);
+  return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+}
+function createTauBenchAdapter(config) {
+  const fixturesMode = process.env[config.fixturesEnv] === "1";
+  return {
+    name: config.name,
+    output: tauResultsOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = benchDir(config);
+      if (!dir) {
+        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`);
+      }
+      await assertPath(`${dir}/src/tau2/registry.py`, "tau registry", config.name);
+      await loadOfficialTasks(config, dir, { limit: 1 });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(config, opts);
+      const dir = benchDir(config);
+      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`);
+      return loadOfficialTasks(config, dir, opts);
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const dir = benchDir(config);
+      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`);
+      const meta = readMeta(task, config.name);
+      const artifactPath = resolve(artifact.trim());
+      await assertPath(artifactPath, "tau results/trajectory artifact", config.name);
+      const report = await scoreOfficialTrajectory(dir, meta, artifactPath);
+      const score = typeof report.score === "number" ? report.score : 0;
+      return {
+        resolved: score === 1,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count })
+      };
+    }
+  };
+}
+
+export {
+  tauResultsOutput,
+  createTauBenchAdapter
+};
+//# sourceMappingURL=chunk-R36V2VP7.js.map