@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/chunk-PA2ZKHJC.js

@@ -1,230 +0,0 @@
-// src/benchmarks/hotpotqa.ts
-import { execFile } from "child_process";
-import { readFile } from "fs/promises";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = join(BENCH_ROOT, ".venv", "bin", "python");
-var FIXTURES = join(BENCH_ROOT, "fixtures", "hotpotqa.json");
-var DATASET = "hotpotqa/hotpot_qa";
-var DATASET_CONFIG = "distractor";
-var DATASET_SPLIT = "validation";
-var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
-var DEFAULT_F1_PASS = 0.6;
-var WORKER_CONTRACT = [
-  "",
-  "Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.",
-  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
-  "The answer after the sentinel must be the bare value only (no explanation on that line)."
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
-    maxBuffer: 1024 * 1024 * 256
-  });
-  return stdout;
-}
-var ARTICLES = /* @__PURE__ */ new Set(["a", "an", "the"]);
-function normalizeAnswer(input) {
-  const lower = input.toLowerCase();
-  const noPunct = lower.replace(/[^\w\s]/g, " ");
-  const tokens = noPunct.split(/\s+/).filter((t) => t.length > 0).filter((t) => !ARTICLES.has(t));
-  return tokens.join(" ").trim();
-}
-function answerTokens(input) {
-  const n = normalizeAnswer(input);
-  return n.length === 0 ? [] : n.split(" ");
-}
-function exactMatch(prediction, gold) {
-  return normalizeAnswer(prediction) === normalizeAnswer(gold);
-}
-function tokenF1(prediction, gold) {
-  const predTokens = answerTokens(prediction);
-  const goldTokens = answerTokens(gold);
-  if (predTokens.length === 0 || goldTokens.length === 0) {
-    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0;
-  }
-  const goldCounts = /* @__PURE__ */ new Map();
-  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1);
-  let common = 0;
-  for (const t of predTokens) {
-    const left = goldCounts.get(t);
-    if (left !== void 0 && left > 0) {
-      common += 1;
-      goldCounts.set(t, left - 1);
-    }
-  }
-  if (common === 0) return 0;
-  const precision = common / predTokens.length;
-  const recall = common / goldTokens.length;
-  return 2 * precision * recall / (precision + recall);
-}
-function f1PassThreshold() {
-  const raw = process.env.HOTPOTQA_F1_PASS;
-  if (raw === void 0 || raw.length === 0) return DEFAULT_F1_PASS;
-  const v = Number(raw);
-  if (!Number.isFinite(v) || v < 0 || v > 1) {
-    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`);
-  }
-  return v;
-}
-function parseFinalAnswer(artifact) {
-  const lines = artifact.split(/\r?\n/);
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const line = lines[i] ?? "";
-    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
-    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
-  }
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const t = (lines[i] ?? "").trim();
-    if (t.length > 0) return t;
-  }
-  return "";
-}
-function rowToTask(row) {
-  const meta = {
-    gold: row.answer,
-    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },
-    type: row.type ?? "",
-    level: row.level ?? "",
-    rawQuestion: row.question
-  };
-  return {
-    id: `hotpotqa-${row.id}`,
-    split: DATASET_SPLIT,
-    prompt: row.question + WORKER_CONTRACT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.gold !== "string") {
-    throw new Error(`HotpotQA task ${task.id} missing metadata.gold \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(
-    `[hotpotqa] HOTPOTQA_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`
-  );
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-function createHotpotqaAdapter() {
-  const fixturesMode = process.env.HOTPOTQA_FIXTURES === "1";
-  f1PassThreshold();
-  return {
-    name: "hotpotqa",
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(
-            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
-          );
-        });
-        return;
-      }
-      try {
-        await py(
-          `from datasets import load_dataset
-load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
-print('ok')`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `hotpotqa preflight failed: ${msg}
-Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; (2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const limit = opts.limit ?? 10;
-      const script = `
-import json, sys
-from datasets import load_dataset
-ds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
-ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
-out = []
-for r in ds:
-    rid = f"hotpotqa-{r['id']}"
-    if ids is not None and rid not in ids:
-        continue
-    sf = r.get('supporting_facts', {}) or {}
-    out.append({
-        "id": r["id"],
-        "question": r.get("question", ""),
-        "answer": r.get("answer", ""),
-        "type": str(r.get("type", "")),
-        "level": str(r.get("level", "")),
-        "supporting_facts": {
-            "title": list(sf.get("title", [])),
-            "sent_id": [int(x) for x in sf.get("sent_id", [])],
-        },
-    })
-    if ids is None and len(out) >= ${limit}:
-        break
-print(json.dumps(out))
-`;
-      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
-      const rows = JSON.parse(stdout);
-      return rows.map(rowToTask);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const finalAnswer = parseFinalAnswer(artifact);
-      if (finalAnswer.length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({
-            reason: "no parseable answer",
-            em: false,
-            f1: 0,
-            normalizedGold: normalizeAnswer(meta.gold)
-          })
-        };
-      }
-      const em = exactMatch(finalAnswer, meta.gold);
-      const f1 = tokenF1(finalAnswer, meta.gold);
-      const threshold = f1PassThreshold();
-      const resolved = em || f1 >= threshold;
-      return {
-        resolved,
-        score: f1,
-        detail: JSON.stringify({
-          em,
-          f1,
-          threshold,
-          normalizedAnswer: normalizeAnswer(finalAnswer),
-          normalizedGold: normalizeAnswer(meta.gold),
-          type: meta.type,
-          level: meta.level,
-          supportingFacts: meta.supportingFacts
-        })
-      };
-    }
-  };
-}
-
-export {
-  normalizeAnswer,
-  exactMatch,
-  tokenF1,
-  parseFinalAnswer,
-  createHotpotqaAdapter
-};
-//# sourceMappingURL=chunk-PA2ZKHJC.js.map

package/dist/chunk-PA2ZKHJC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/hotpotqa.ts"],"sourcesContent":["/**\n * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').\n * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.\n *\n * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no\n * LLM. Both the predicted final answer and the gold are normalized (lowercase,\n * strip articles a/an/the, strip punctuation, collapse whitespace), then scored\n * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS\n * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the\n * suite a judge that needs no model tokens at all.\n *\n * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of\n * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,\n * never part of the score.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed + network\n * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the\n * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'hotpotqa.json')\n\nconst DATASET = 'hotpotqa/hotpot_qa'\nconst DATASET_CONFIG = 'distractor'\nconst DATASET_SPLIT = 'validation'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Default F1 pass threshold; HotpotQA's leaderboard reports EM and F1 separately. */\nconst DEFAULT_F1_PASS = 0.6\n\n/** Worker contract appended to every task prompt. The judge keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\ninterface SupportingFacts {\n  title: string[]\n  sent_id: number[]\n}\n\ninterface HotpotRow {\n  id: string\n  question: string\n  answer: string\n  type: string\n  level: string\n  supporting_facts: SupportingFacts\n}\n\ninterface HotpotMeta {\n  gold: string\n  supportingFacts: SupportingFacts\n  type: string\n  level: string\n  rawQuestion: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\n\n/**\n * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,\n * drop articles (a/an/the), collapse whitespace. Token comparisons run on the\n * output of this exactly as the published evaluator does.\n */\nexport function normalizeAnswer(input: string): string {\n  const lower = input.toLowerCase()\n  // strip punctuation: keep word chars + whitespace only\n  const noPunct = lower.replace(/[^\\w\\s]/g, ' ')\n  const tokens = noPunct\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Normalized-token list (the F1 bag-of-tokens unit). */\nfunction answerTokens(input: string): string[] {\n  const n = normalizeAnswer(input)\n  return n.length === 0 ? [] : n.split(' ')\n}\n\n/** Exact match on the normalized strings. */\nexport function exactMatch(prediction: string, gold: string): boolean {\n  return normalizeAnswer(prediction) === normalizeAnswer(gold)\n}\n\n/**\n * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of\n * normalized tokens. Mirrors the published evaluator's special-case handling of\n * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both\n * are empty, else 0.\n */\nexport function tokenF1(prediction: string, gold: string): number {\n  const predTokens = answerTokens(prediction)\n  const goldTokens = answerTokens(gold)\n  if (predTokens.length === 0 || goldTokens.length === 0) {\n    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const goldCounts = new Map<string, number>()\n  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1)\n  let common = 0\n  for (const t of predTokens) {\n    const left = goldCounts.get(t)\n    if (left !== undefined && left > 0) {\n      common += 1\n      goldCounts.set(t, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / predTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\n/** Read the configured F1 pass threshold; fail loud on a malformed override. */\nfunction f1PassThreshold(): number {\n  const raw = process.env.HOTPOTQA_F1_PASS\n  if (raw === undefined || raw.length === 0) return DEFAULT_F1_PASS\n  const v = Number(raw)\n  if (!Number.isFinite(v) || v < 0 || v > 1) {\n    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`)\n  }\n  return v\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the trimmed last non-empty line. Returns '' when nothing is\n * parseable (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\nfunction rowToTask(row: HotpotRow): BenchTask {\n  const meta: HotpotMeta = {\n    gold: row.answer,\n    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },\n    type: row.type ?? '',\n    level: row.level ?? '',\n    rawQuestion: row.question,\n  }\n  return {\n    id: `hotpotqa-${row.id}`,\n    split: DATASET_SPLIT,\n    prompt: row.question + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): HotpotMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`HotpotQA task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as HotpotMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as HotpotRow[]\n  console.warn(\n    `[hotpotqa] HOTPOTQA_FIXTURES=1 — loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`,\n  )\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createHotpotqaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.HOTPOTQA_FIXTURES === '1'\n  // Validate the threshold at construction so a malformed env fails before any run.\n  f1PassThreshold()\n\n  return {\n    name: 'hotpotqa',\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `hotpotqa preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; ` +\n            `or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    rid = f\"hotpotqa-{r['id']}\"\n    if ids is not None and rid not in ids:\n        continue\n    sf = r.get('supporting_facts', {}) or {}\n    out.append({\n        \"id\": r[\"id\"],\n        \"question\": r.get(\"question\", \"\"),\n        \"answer\": r.get(\"answer\", \"\"),\n        \"type\": str(r.get(\"type\", \"\")),\n        \"level\": str(r.get(\"level\", \"\")),\n        \"supporting_facts\": {\n            \"title\": list(sf.get(\"title\", [])),\n            \"sent_id\": [int(x) for x in sf.get(\"sent_id\", [])],\n        },\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as HotpotRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            reason: 'no parseable answer',\n            em: false,\n            f1: 0,\n            normalizedGold: normalizeAnswer(meta.gold),\n          }),\n        }\n      }\n\n      const em = exactMatch(finalAnswer, meta.gold)\n      const f1 = tokenF1(finalAnswer, meta.gold)\n      const threshold = f1PassThreshold()\n      const resolved = em || f1 >= threshold\n      return {\n        resolved,\n        score: f1,\n        detail: JSON.stringify({\n          em,\n          f1,\n          threshold,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          type: meta.type,\n          level: meta.level,\n          supportingFacts: meta.supportingFacts,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AAoBA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,IAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,IAAM,UAAU;AAChB,IAAM,iBAAiB;AACvB,IAAM,gBAAgB;AACtB,IAAM,wBAAwB;AAG9B,IAAM,kBAAkB;AAGxB,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA,iDAAiD,qBAAqB;AAAA,EACtE;AACF,EAAE,KAAK,IAAI;AAyBX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IAClE,WAAW,OAAO,OAAO;AAAA,EAC3B,CAAC;AACD,SAAO;AACT;AAEA,IAAM,WAAW,oBAAI,IAAI,CAAC,KAAK,MAAM,KAAK,CAAC;AAOpC,SAAS,gBAAgB,OAAuB;AACrD,QAAM,QAAQ,MAAM,YAAY;AAEhC,QAAM,UAAU,MAAM,QAAQ,YAAY,GAAG;AAC7C,QAAM,SAAS,QACZ,MAAM,KAAK,EACX,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;AACjC,SAAO,OAAO,KAAK,GAAG,EAAE,KAAK;AAC/B;AAGA,SAAS,aAAa,OAAyB;AAC7C,QAAM,IAAI,gBAAgB,KAAK;AAC/B,SAAO,EAAE,WAAW,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;AAC1C;AAGO,SAAS,WAAW,YAAoB,MAAuB;AACpE,SAAO,gBAAgB,UAAU,MAAM,gBAAgB,IAAI;AAC7D;AAQO,SAAS,QAAQ,YAAoB,MAAsB;AAChE,QAAM,aAAa,aAAa,UAAU;AAC1C,QAAM,aAAa,aAAa,IAAI;AACpC,MAAI,WAAW,WAAW,KAAK,WAAW,WAAW,GAAG;AACtD,WAAO,WAAW,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;AAAA,EAClE;AACA,QAAM,aAAa,oBAAI,IAAoB;AAC3C,aAAW,KAAK,WAAY,YAAW,IAAI,IAAI,WAAW,IAAI,CAAC,KAAK,KAAK,CAAC;AAC1E,MAAI,SAAS;AACb,aAAW,KAAK,YAAY;AAC1B,UAAM,OAAO,WAAW,IAAI,CAAC;AAC7B,QAAI,SAAS,UAAa,OAAO,GAAG;AAClC,gBAAU;AACV,iBAAW,IAAI,GAAG,OAAO,CAAC;AAAA,IAC5B;AAAA,EACF;AACA,MAAI,WAAW,EAAG,QAAO;AACzB,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,SAAS,SAAS,WAAW;AACnC,SAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAGA,SAAS,kBAA0B;AACjC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,QAAQ,UAAa,IAAI,WAAW,EAAG,QAAO;AAClD,QAAM,IAAI,OAAO,GAAG;AACpB,MAAI,CAAC,OAAO,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AACzC,UAAM,IAAI,MAAM,mDAAmD,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EAC1F;AACA,SAAO;AACT;AAQO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,KAAK,MAAM,CAAC,KAAK,IAAI,KAAK;AAChC,QAAI,EAAE,SAAS,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAEA,SAAS,UAAU,KAA2B;AAC5C,QAAM,OAAmB;AAAA,IACvB,MAAM,IAAI;AAAA,IACV,iBAAiB,IAAI,oBAAoB,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClE,MAAM,IAAI,QAAQ;AAAA,IAClB,OAAO,IAAI,SAAS;AAAA,IACpB,aAAa,IAAI;AAAA,EACnB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,IAAI,EAAE;AAAA,IACtB,OAAO;AAAA,IACP,QAAQ,IAAI,WAAW;AAAA,IACvB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA6B;AAC7C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAAU;AACtC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,6DAAwD;AAAA,EAClG;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,iDAA4C,KAAK,MAAM,4BAA4B,QAAQ;AAAA,EAC7F;AACA,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,kBAAgB;AAEhB,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,2BAA2B,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAC7F;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,eACK,KAAK,UAAU,OAAO,CAAC,KAAK,KAAK,UAAU,cAAc,CAAC,WAAW,KAAK,UAAU,aAAa,CAAC;AAAA;AAAA,QAEzG;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,+HAEkB,OAAO,KAAK,cAAc;AAAA,QAE/E;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,SAAS;AAAA;AAAA;AAAA,oBAGD,KAAK,UAAU,OAAO,CAAC,KAAK,KAAK,UAAU,cAAc,CAAC,WAAW,KAAK,UAAU,aAAa,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAmBjF,KAAK;AAAA;AAAA;AAAA;AAIpC,YAAM,SAAS,MAAM,GAAG,QAAQ,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;AAC1E,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK,IAAI,SAAS;AAAA,IAC3B;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,GAAG,qBAAqB,IAAI,KAAK,IAAI;AAAA,IAC9C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,cAAc,iBAAiB,QAAQ;AAE7C,UAAI,YAAY,WAAW,GAAG;AAE5B,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU;AAAA,YACrB,QAAQ;AAAA,YACR,IAAI;AAAA,YACJ,IAAI;AAAA,YACJ,gBAAgB,gBAAgB,KAAK,IAAI;AAAA,UAC3C,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,KAAK,WAAW,aAAa,KAAK,IAAI;AAC5C,YAAM,KAAK,QAAQ,aAAa,KAAK,IAAI;AACzC,YAAM,YAAY,gBAAgB;AAClC,YAAM,WAAW,MAAM,MAAM;AAC7B,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,QACP,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA;AAAA,UACA,kBAAkB,gBAAgB,WAAW;AAAA,UAC7C,gBAAgB,gBAAgB,KAAK,IAAI;AAAA,UACzC,MAAM,KAAK;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,iBAAiB,KAAK;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-PUIRNYI7.js

@@ -1,189 +0,0 @@
-// src/benchmarks/cadbench.ts
-import { readFile as readFile2 } from "fs/promises";
-
-// src/worker-blender.ts
-import { execFile } from "child_process";
-import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { promisify } from "util";
-import { routerChatWithUsage } from "@tangle-network/agent-runtime/loops";
-var execFileAsync = promisify(execFile);
-async function runLocal(cmd, args, cwd, timeoutMs = 18e4) {
-  try {
-    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs });
-    return { code: 0, stdout, stderr };
-  } catch (err) {
-    const e = err;
-    return { code: typeof e.code === "number" ? e.code : 1, stdout: e.stdout ?? "", stderr: e.stderr ?? e.message ?? String(err) };
-  }
-}
-var RUNNER_PY = `
-import bpy, sys, math, mathutils, traceback, os
-agent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])
-bpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()
-ok=True
-try:
-    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}
-    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)
-except Exception as e:
-    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False
-meshes=[o for o in bpy.context.scene.objects if o.type=='MESH']
-if not meshes:
-    print('NO_MESH'); sys.exit(0 if ok else 3)
-mn=[1e18]*3; mx=[-1e18]*3
-for o in meshes:
-    for c in o.bound_box:
-        w=o.matrix_world @ mathutils.Vector(c)
-        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])
-center=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))
-size=max(mx[i]-mn[i] for i in range(3)) or 1.0
-# standardize: drop any agent-added cameras/lights
-for o in list(bpy.context.scene.objects):
-    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)
-w=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w
-w.use_nodes=True
-try: w.node_tree.nodes['Background'].inputs[1].default_value=0.6
-except Exception: pass
-bpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))
-bpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam
-sc=bpy.context.scene
-sc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False
-sc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False
-dist=size*2.4
-el=math.radians(58)
-for v in range(nviews):
-    az=math.radians(40 + v*360.0/nviews)
-    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist
-    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()
-    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)
-print('RENDER_DONE')
-`.trim();
-async function renderBpy(script, opts = {}) {
-  const views = Math.max(1, opts.views ?? 4);
-  const dir = await mkdtemp(join(tmpdir(), "blender-judge-"));
-  const runnerPath = join(dir, "runner.py");
-  const scriptPath = join(dir, "model.py");
-  try {
-    await writeFile(runnerPath, RUNNER_PY);
-    await writeFile(scriptPath, script);
-    const run = await runLocal("xvfb-run", ["-a", "blender", "--background", "--python", runnerPath, "--", scriptPath, dir, String(views)], dir);
-    const out = `${run.stdout}
-${run.stderr}`;
-    const built = /RENDER_DONE/.test(out);
-    if (!built) return { built: false, renders: [], error: /(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400) };
-    const renders = [];
-    for (let v = 0; v < views; v++) {
-      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => void 0);
-      if (buf) renders.push(`data:image/png;base64,${buf.toString("base64")}`);
-    }
-    return { built: renders.length > 0, renders };
-  } finally {
-    await rm(dir, { recursive: true, force: true }).catch(() => {
-    });
-  }
-}
-
-// src/benchmarks/cadbench.ts
-function must(name) {
-  const v = process.env[name];
-  if (!v) throw new Error(`env ${name} is required for the CADBench judge`);
-  return v;
-}
-async function judgeCriteria(instruction, script, criteria, renders) {
-  const base = (process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1").replace(/\/$/, "");
-  const key = must("TANGLE_API_KEY");
-  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
-  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join("\n");
-  const text = `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:
-"${instruction}"
-
-Below are ${renders.length} rendered views of the produced model, and the script that built it. For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.
-
-CRITERIA:
-${numbered}
-
-SCRIPT:
-\`\`\`python
-${script.slice(0, 6e3)}
-\`\`\``;
-  const content = [{ type: "text", text }];
-  for (const url of renders) content.push({ type: "image_url", image_url: { url } });
-  const res = await fetch(`${base}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
-    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: "user", content }] })
-  });
-  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`);
-  const data = await res.json();
-  const raw = data.choices?.[0]?.message?.content ?? "";
-  const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw);
-  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` };
-  let arr;
-  try {
-    arr = JSON.parse(m[0].toLowerCase());
-  } catch {
-    return { passed: criteria.map(() => false), note: "judge verdict not valid JSON" };
-  }
-  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : [];
-  const passed = criteria.map((_, i) => bools[i] === true);
-  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` };
-}
-function createCadBenchAdapter() {
-  let cache = null;
-  async function load() {
-    if (cache) return cache;
-    const path = process.env.CADBENCH_PATH;
-    if (!path) throw new Error("CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)");
-    const text = await readFile2(path, "utf8");
-    cache = text.split("\n").filter((l) => l.trim()).map((l) => {
-      const r = JSON.parse(l);
-      return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? "", type: r.type ?? "", criteria: r.criteria } };
-    });
-    return cache;
-  }
-  return {
-    name: "cadbench",
-    async preflight() {
-      const { execFile: execFile2 } = await import("child_process");
-      const { promisify: promisify2 } = await import("util");
-      const exec = promisify2(execFile2);
-      try {
-        await exec("xvfb-run", ["-a", "blender", "--version"], { timeout: 3e4 });
-      } catch (err) {
-        throw new Error(
-          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}
-Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \`xvfb-run -a blender --background --python\`.`
-        );
-      }
-      await load();
-    },
-    async loadTasks(opts = {}) {
-      let rows = await load();
-      if (opts.ids) rows = rows.filter((r) => opts.ids.includes(r.id));
-      const t = process.env.CADBENCH_TYPE;
-      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase());
-      if (opts.limit != null) rows = rows.slice(0, opts.limit);
-      return rows.map((r) => ({ id: r.id, prompt: r.instruction, metadata: r.meta }));
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = task.metadata;
-      const criteria = meta.criteria ?? [];
-      if (!artifact.trim()) return { resolved: false, score: 0, detail: "empty artifact" };
-      if (criteria.length === 0) return { resolved: false, score: 0, detail: "task has no criteria" };
-      const r = await renderBpy(artifact, { views: 4 });
-      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? "no mesh"}` };
-      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders);
-      const score = passed.filter(Boolean).length / criteria.length;
-      return { resolved: score === 1, score, detail: note };
-    }
-  };
-}
-
-export {
-  createCadBenchAdapter
-};
-//# sourceMappingURL=chunk-PUIRNYI7.js.map

package/dist/chunk-PUIRNYI7.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/cadbench.ts","../src/worker-blender.ts"],"sourcesContent":["/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const res = await fetch(`${base}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n  })\n  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n  const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const raw = data.choices?.[0]?.message?.content ?? ''\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n","/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const { content, usage: u } = await routerChatWithUsage(\n        cfg,\n        [\n          { role: 'system', content: directive },\n          { role: 'user', content: user },\n        ],\n        { temperature: 0.3 },\n      )\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n"],"mappings":";AAaA,SAAS,YAAAA,iBAAgB;;;ACFzB,SAAS,gBAAgB;AACzB,SAAS,SAAS,UAAU,IAAI,iBAAiB;AACjD,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAK1B,SAAS,2BAA2B;AAIpC,IAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;AAChJ,MAAI;AACF,UAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,cAAc,KAAK,MAAM,EAAE,KAAK,WAAW,KAAK,IAAI,SAAS,UAAU,CAAC;AACzG,WAAO,EAAE,MAAM,GAAG,QAAQ,OAAO;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,WAAO,EAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO,GAAG,QAAQ,EAAE,UAAU,IAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,EAAE;AAAA,EAC/H;AACF;AAcA,IAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAwChB,KAAK;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;AAC7I,QAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;AACzC,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;AAC1D,QAAM,aAAa,KAAK,KAAK,WAAW;AACxC,QAAM,aAAa,KAAK,KAAK,UAAU;AACvC,MAAI;AACF,UAAM,UAAU,YAAY,SAAS;AACrC,UAAM,UAAU,YAAY,MAAM;AAClC,UAAM,MAAM,MAAM,SAAS,YAAY,CAAC,MAAM,WAAW,gBAAgB,YAAY,YAAY,MAAM,YAAY,KAAK,OAAO,KAAK,CAAC,GAAG,GAAG;AAC3I,UAAM,MAAM,GAAG,IAAI,MAAM;AAAA,EAAK,IAAI,MAAM;AACxC,UAAM,QAAQ,cAAc,KAAK,GAAG;AACpC,QAAI,CAAC,MAAO,QAAO,EAAE,OAAO,OAAO,SAAS,CAAC,GAAG,OAAQ,kCAAkC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,MAAM,IAAI,EAAG;AACpI,UAAM,UAAoB,CAAC;AAC3B,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,MAAM,MAAS;AAC5E,UAAI,IAAK,SAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,CAAC,EAAE;AAAA,IACzE;AACA,WAAO,EAAE,OAAO,QAAQ,SAAS,GAAG,QAAQ;AAAA,EAC9C,UAAE;AACA,UAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAChE;AACF;;;AD1FA,SAAS,KAAK,MAAsB;AAClC,QAAM,IAAI,QAAQ,IAAI,IAAI;AAC1B,MAAI,CAAC,EAAG,OAAM,IAAI,MAAM,OAAO,IAAI,qCAAqC;AACxE,SAAO;AACT;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;AAC9C,QAAM,QAAQ,QAAQ,IAAI,eAAe,kCAAkC,QAAQ,OAAO,EAAE;AAC5F,QAAM,MAAM,KAAK,gBAAgB;AACjC,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,WAAW,SAAS,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,IAAI;AACnE,QAAM,OACJ;AAAA,GAAsG,WAAW;AAAA;AAAA,YACpG,QAAQ,MAAM,8TAEY,SAAS,MAAM;AAAA;AAAA;AAAA,EAA4E,QAAQ;AAAA;AAAA;AAAA;AAAA,EAA8B,OAAO,MAAM,GAAG,GAAI,CAAC;AAAA;AAC/L,QAAM,UAAqB,CAAC,EAAE,MAAM,QAAQ,KAAK,CAAC;AAClD,aAAW,OAAO,QAAS,SAAQ,KAAK,EAAE,MAAM,aAAa,WAAW,EAAE,IAAI,EAAE,CAAC;AACjF,QAAM,MAAM,MAAM,MAAM,GAAG,IAAI,qBAAqB;AAAA,IAClD,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,GAAG,GAAG;AAAA,IAC9E,MAAM,KAAK,UAAU,EAAE,OAAO,YAAY,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,MAAM,QAAQ,QAAQ,CAAC,EAAE,CAAC;AAAA,EACzG,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,SAAS,KAAK,IAAI,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAChG,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,MAAM,KAAK,UAAU,CAAC,GAAG,SAAS,WAAW;AACnD,QAAM,IAAI,iCAAiC,KAAK,GAAG;AACnD,MAAI,CAAC,EAAG,QAAO,EAAE,QAAQ,SAAS,IAAI,MAAM,KAAK,GAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE,CAAC,GAAG;AACrH,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,MAAM,EAAE,CAAC,EAAE,YAAY,CAAC;AAAA,EACrC,QAAQ;AACN,WAAO,EAAE,QAAQ,SAAS,IAAI,MAAM,KAAK,GAAG,MAAM,+BAA+B;AAAA,EACnF;AACA,QAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,MAAM,IAAI,IAAI,CAAC;AAEjE,QAAM,SAAS,SAAS,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,MAAM,IAAI;AACvD,SAAO,EAAE,QAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,EAAE,MAAM,IAAI,SAAS,MAAM,YAAY;AACxF;AAEO,SAAS,wBAA0C;AACxD,MAAI,QAA+E;AAEnF,iBAAe,OAAuC;AACpD,QAAI,MAAO,QAAO;AAClB,UAAM,OAAO,QAAQ,IAAI;AACzB,QAAI,CAAC,KAAM,OAAM,IAAI,MAAM,qGAAqG;AAChI,UAAM,OAAO,MAAMC,UAAS,MAAM,MAAM;AACxC,YAAQ,KACL,MAAM,IAAI,EACV,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EACtB,IAAI,CAAC,MAAM;AACV,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,aAAO,EAAE,IAAI,EAAE,IAAI,aAAa,EAAE,aAAa,MAAM,EAAE,MAAM,EAAE,QAAQ,IAAI,MAAM,EAAE,QAAQ,IAAI,UAAU,EAAE,SAAS,EAAE;AAAA,IACxH,CAAC;AACH,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,EAAE,UAAAC,UAAS,IAAI,MAAM,OAAO,eAAoB;AACtD,YAAM,EAAE,WAAAC,WAAU,IAAI,MAAM,OAAO,MAAW;AAC9C,YAAM,OAAOA,WAAUD,SAAQ;AAC/B,UAAI;AACF,cAAM,KAAK,YAAY,CAAC,MAAM,WAAW,WAAW,GAAG,EAAE,SAAS,IAAO,CAAC;AAAA,MAC5E,SAAS,KAAK;AACZ,cAAM,IAAI;AAAA,UACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAAG,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA,QAEhG;AAAA,MACF;AACA,YAAM,KAAK;AAAA,IACb;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,OAAO,MAAM,KAAK;AACtB,UAAI,KAAK,IAAK,QAAO,KAAK,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAEhE,YAAM,IAAI,QAAQ,IAAI;AACtB,UAAI,EAAG,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;AAC9E,UAAI,KAAK,SAAS,KAAM,QAAO,KAAK,MAAM,GAAG,KAAK,KAAK;AACvD,aAAO,KAAK,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,aAAa,UAAU,EAAE,KAA2C,EAAE;AAAA,IACjI;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,KAAK;AAClB,YAAM,WAAW,KAAK,YAAY,CAAC;AACnC,UAAI,CAAC,SAAS,KAAK,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,iBAAiB;AACnF,UAAI,SAAS,WAAW,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,uBAAuB;AAC9F,YAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;AAChD,UAAI,CAAC,EAAE,MAAO,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,yBAAyB,EAAE,SAAS,SAAS,GAAG;AAC1G,YAAM,EAAE,QAAQ,KAAK,IAAI,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;AACvF,YAAM,QAAQ,OAAO,OAAO,OAAO,EAAE,SAAS,SAAS;AACvD,aAAO,EAAE,UAAU,UAAU,GAAG,OAAO,QAAQ,KAAK;AAAA,IACtD;AAAA,EACF;AACF;","names":["readFile","readFile","execFile","promisify"]}

package/dist/chunk-QZZEAHWJ.js

@@ -1,136 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  runVenvScriptStdin
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/programbench.ts
-import { join } from "path";
-import { readFile } from "fs/promises";
-var FIXTURES = join(benchRoot, "fixtures", "programbench.json");
-var TESTS_DATASET = "programbench/ProgramBench-Tests";
-var TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`;
-var PROMPT = [
-  "This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.",
-  "Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.",
-  "You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.",
-  "Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence."
-].join("\n");
-function fixtureToTask(f) {
-  const meta = { instanceId: f.instance_id, imageName: f.image_name };
-  return {
-    id: f.instance_id,
-    prompt: PROMPT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.instanceId !== "string") {
-    throw new Error(`programbench task ${task.id} missing metadata.instanceId \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectTasks(fixtures, opts) {
-  let tasks = fixtures.map(fixtureToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-function imageName(instanceId) {
-  return instanceId.split(".")[0].replace("__", "/");
-}
-async function loadFixtures(opts) {
-  const fixtures = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(
-    `[programbench] PROGRAMBENCH_FIXTURES=1 \u2014 loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`
-  );
-  return selectTasks(fixtures, opts);
-}
-async function fetchInstanceIds() {
-  const res = await fetch(TREE_API);
-  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`);
-  const tree = await res.json();
-  const ids = tree.filter((e) => e.type === "directory" && e.path.includes("__")).map((e) => e.path);
-  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`);
-  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }));
-}
-async function runHarness(meta, submission) {
-  const judge = join(benchRoot, "scripts", "programbench_judge.py");
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(judge, ["--instance", meta.instanceId], submission, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
-  }
-  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`);
-  if (typeof report.passed !== "number" || typeof report.total !== "number") {
-    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
-  }
-  const score = report.total > 0 ? report.passed / report.total : 0;
-  return {
-    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),
-    score,
-    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total })
-  };
-}
-var programbenchSubmissionOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const blocks = [...text.matchAll(/```path:([^\n`]+)\n([\s\S]*?)```/g)];
-    if (blocks.length === 0) return "";
-    return blocks.map((m) => `===FILE:${m[1].trim()}===
-${m[2]}`).join("\n");
-  }
-};
-function createProgrambenchAdapter() {
-  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === "1";
-  return {
-    name: "programbench",
-    output: programbenchSubmissionOutput,
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["programbench"],
-        requireDocker: true,
-        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; (2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; (3) HF access for the hidden test blobs (the driver runs \`programbench blob sync\`). Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`
-      });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      let fixtures;
-      try {
-        fixtures = await fetchInstanceIds();
-      } catch (err) {
-        console.warn(
-          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
-        );
-        return loadFixtures(opts);
-      }
-      return selectTasks(fixtures, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      return runHarness(meta, artifact);
-    }
-  };
-}
-
-export {
-  programbenchSubmissionOutput,
-  createProgrambenchAdapter
-};
-//# sourceMappingURL=chunk-QZZEAHWJ.js.map