@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/cadbench-DpQWZHp4.js

@@ -0,0 +1,285 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { tmpdir } from "node:os";
+import { promisify } from "node:util";
+import "@tangle-network/agent-runtime/loops";
+//#region src/worker-blender.ts
+/**
+* BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender
+* `bpy` Python script that builds the described 3D model. We author it via the
+* router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the
+* produced geometry, and render N standardized views — the images the CADBench
+* criteria judge scores. The authoring directive is the GEPA-optimizable surface.
+*
+* Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU
+* with denoising off (the apt build ships without OpenImageDenoise).
+*/
+const execFileAsync = promisify(execFile);
+async function runLocal(cmd, args, cwd, timeoutMs = 18e4) {
+	try {
+		const { stdout, stderr } = await execFileAsync(cmd, args, {
+			cwd,
+			maxBuffer: 1 << 26,
+			timeout: timeoutMs
+		});
+		return {
+			code: 0,
+			stdout,
+			stderr
+		};
+	} catch (err) {
+		const e = err;
+		return {
+			code: typeof e.code === "number" ? e.code : 1,
+			stdout: e.stdout ?? "",
+			stderr: e.stderr ?? e.message ?? String(err)
+		};
+	}
+}
+/**
+* The standardized Blender runner (written to a temp file per run). It clears the
+* scene, executes the agent's bpy script, auto-frames the produced meshes, sets
+* up neutral lighting, and renders N azimuth views with Cycles CPU.
+*/
+const RUNNER_PY = `
+import bpy, sys, math, mathutils, traceback, os
+agent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])
+bpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()
+ok=True
+try:
+    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}
+    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)
+except Exception as e:
+    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False
+meshes=[o for o in bpy.context.scene.objects if o.type=='MESH']
+if not meshes:
+    print('NO_MESH'); sys.exit(0 if ok else 3)
+mn=[1e18]*3; mx=[-1e18]*3
+for o in meshes:
+    for c in o.bound_box:
+        w=o.matrix_world @ mathutils.Vector(c)
+        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])
+center=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))
+size=max(mx[i]-mn[i] for i in range(3)) or 1.0
+# standardize: drop any agent-added cameras/lights
+for o in list(bpy.context.scene.objects):
+    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)
+w=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w
+w.use_nodes=True
+try: w.node_tree.nodes['Background'].inputs[1].default_value=0.6
+except Exception: pass
+bpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))
+bpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam
+sc=bpy.context.scene
+sc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False
+sc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False
+dist=size*2.4
+el=math.radians(58)
+for v in range(nviews):
+    az=math.radians(40 + v*360.0/nviews)
+    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist
+    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()
+    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)
+print('RENDER_DONE')
+`.trim();
+/** Execute a bpy script headless + render N standardized views — no authoring.
+*  Used by the CADBench judge to render an artifact before vision-scoring it. */
+async function renderBpy(script, opts = {}) {
+	const views = Math.max(1, opts.views ?? 4);
+	const dir = await mkdtemp(join(tmpdir(), "blender-judge-"));
+	const runnerPath = join(dir, "runner.py");
+	const scriptPath = join(dir, "model.py");
+	try {
+		await writeFile(runnerPath, RUNNER_PY);
+		await writeFile(scriptPath, script);
+		const run = await runLocal("xvfb-run", [
+			"-a",
+			"blender",
+			"--background",
+			"--python",
+			runnerPath,
+			"--",
+			scriptPath,
+			dir,
+			String(views)
+		], dir);
+		const out = `${run.stdout}\n${run.stderr}`;
+		if (!/RENDER_DONE/.test(out)) return {
+			built: false,
+			renders: [],
+			error: /(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)
+		};
+		const renders = [];
+		for (let v = 0; v < views; v++) {
+			const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => void 0);
+			if (buf) renders.push(`data:image/png;base64,${buf.toString("base64")}`);
+		}
+		return {
+			built: renders.length > 0,
+			renders
+		};
+	} finally {
+		await rm(dir, {
+			recursive: true,
+			force: true
+		}).catch(() => {});
+	}
+}
+//#endregion
+//#region src/benchmarks/cadbench.ts
+/**
+* CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).
+* Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria
+* eval: render the produced model to standardized views, then a vision judge
+* (GPT-4o-class) marks each per-task criterion bullet pass/fail against the
+* rendered images + the script text. score = fraction of criteria satisfied.
+*
+* Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,
+* 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL
+* ({id,name,instruction,type,criteria:string[]} per line). Judge creds from
+* TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).
+*/
+function must(name) {
+	const v = process.env[name];
+	if (!v) throw new Error(`env ${name} is required for the CADBench judge`);
+	return v;
+}
+/** One batched vision call: rendered views + the bpy script + the numbered
+*  criteria → a JSON array of booleans (true = satisfied). Faithful to the
+*  paper's combined image+script evaluation. Throws on transport failure (never
+*  a silent zero); a parse miss falls back to "all fail" with a note. */
+async function judgeCriteria(instruction, script, criteria, renders) {
+	const base = (process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1").replace(/\/$/, "");
+	const key = must("TANGLE_API_KEY");
+	const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+	const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join("\n");
+	const content = [{
+		type: "text",
+		text: `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\n"${instruction}"\n\nBelow are ${renders.length} rendered views of the produced model, and the script that built it. For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\n\nCRITERIA:\n${numbered}\n\nSCRIPT:\n\`\`\`python\n${script.slice(0, 6e3)}\n\`\`\``
+	}];
+	for (const url of renders) content.push({
+		type: "image_url",
+		image_url: { url }
+	});
+	const res = await fetch(`${base}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json",
+			authorization: `Bearer ${key}`
+		},
+		body: JSON.stringify({
+			model,
+			max_tokens: 1500,
+			temperature: 0,
+			messages: [{
+				role: "user",
+				content
+			}]
+		})
+	});
+	if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`);
+	const raw = (await res.json()).choices?.[0]?.message?.content ?? "";
+	const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw);
+	if (!m) return {
+		passed: criteria.map(() => false),
+		note: `judge returned no parseable verdict: ${raw.slice(0, 80)}`
+	};
+	let arr;
+	try {
+		arr = JSON.parse(m[0].toLowerCase());
+	} catch {
+		return {
+			passed: criteria.map(() => false),
+			note: "judge verdict not valid JSON"
+		};
+	}
+	const bools = Array.isArray(arr) ? arr.map((x) => x === true) : [];
+	const passed = criteria.map((_, i) => bools[i] === true);
+	return {
+		passed,
+		note: `${passed.filter(Boolean).length}/${criteria.length} criteria`
+	};
+}
+function createCadBenchAdapter() {
+	let cache = null;
+	async function load() {
+		if (cache) return cache;
+		const path = process.env.CADBENCH_PATH;
+		if (!path) throw new Error("CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)");
+		cache = (await readFile(path, "utf8")).split("\n").filter((l) => l.trim()).map((l) => {
+			const r = JSON.parse(l);
+			return {
+				id: r.id,
+				instruction: r.instruction,
+				meta: {
+					name: r.name ?? "",
+					type: r.type ?? "",
+					criteria: r.criteria
+				}
+			};
+		});
+		return cache;
+	}
+	return {
+		name: "cadbench",
+		async preflight() {
+			const { execFile } = await import("node:child_process");
+			const { promisify } = await import("node:util");
+			const exec = promisify(execFile);
+			try {
+				await exec("xvfb-run", [
+					"-a",
+					"blender",
+					"--version"
+				], { timeout: 3e4 });
+			} catch (err) {
+				throw new Error(`cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\nFix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \`xvfb-run -a blender --background --python\`.`);
+			}
+			await load();
+		},
+		async loadTasks(opts = {}) {
+			let rows = await load();
+			if (opts.ids) rows = rows.filter((r) => opts.ids.includes(r.id));
+			const t = process.env.CADBENCH_TYPE;
+			if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase());
+			if (opts.limit != null) rows = rows.slice(0, opts.limit);
+			return rows.map((r) => ({
+				id: r.id,
+				prompt: r.instruction,
+				metadata: r.meta
+			}));
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			const criteria = task.metadata.criteria ?? [];
+			if (!artifact.trim()) return {
+				resolved: false,
+				score: 0,
+				detail: "empty artifact"
+			};
+			if (criteria.length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: "task has no criteria"
+			};
+			const r = await renderBpy(artifact, { views: 4 });
+			if (!r.built) return {
+				resolved: false,
+				score: 0,
+				detail: `did not build/render: ${r.error ?? "no mesh"}`
+			};
+			const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders);
+			const score = passed.filter(Boolean).length / criteria.length;
+			return {
+				resolved: score === 1,
+				score,
+				detail: note
+			};
+		}
+	};
+}
+//#endregion
+export { createCadBenchAdapter as t };
+
+//# sourceMappingURL=cadbench-DpQWZHp4.js.map

package/dist/cadbench-DpQWZHp4.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadbench-DpQWZHp4.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const { content, usage: u } = await routerChatWithUsage(\n        cfg,\n        [\n          { role: 'system', content: directive },\n          { role: 'user', content: user },\n        ],\n        { temperature: 0.3 },\n      )\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const res = await fetch(`${base}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n  })\n  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n  const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const raw = data.choices?.[0]?.message?.content ?? ''\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;AC1FA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CACjF,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,oBAAoB;EAClD,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU;EAAM;EAC9E,MAAM,KAAK,UAAU;GAAE;GAAO,YAAY;GAAM,aAAa;GAAG,UAAU,CAAC;IAAE,MAAM;IAAQ;GAAQ,CAAC;EAAE,CAAC;CACzG,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAEhG,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,UAAU,EAAE,EAAE,SAAS,WAAW;CACnD,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}

package/dist/cadgenbench-DRhczfsG.js

@@ -0,0 +1,151 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import "node:fs";
+import { tmpdir } from "node:os";
+import { promisify } from "node:util";
+import "@tangle-network/agent-runtime/loops";
+promisify(execFile);
+const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? "/tmp/cgb-venv/bin/python";
+const CGB_DIR = process.env.CADGENBENCH_DIR ?? "/tmp/cadgenbench";
+//#endregion
+//#region src/benchmarks/cadgenbench.ts
+/**
+* CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part
+* description → a STEP B-rep solid (output.step). Score = the benchmark's OWN
+* deterministic geometric metric (cad_score): validity gate → PCA/ICP align to
+* the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.
+* NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.
+*
+* The official task set (private GT, server-side graded) isn't released yet, so
+* tasks here are seeded from the repo's dimension-named geometry fixtures (real
+* GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap
+* loadTasks to read the published fixtures' description.yaml + ground_truth.step.
+*
+* Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +
+* xvfb (the scorer's alignment renders need a display).
+*/
+const execFileAsync = promisify(execFile);
+/** Self-contained scorer wrapper (written to a temp file, run in the venv).
+*  Scores a candidate STEP against a ground-truth STEP via the benchmark's own
+*  evaluate_result, printing the cad_score line. */
+const SCORE_PY = `
+import sys, json, tempfile, shutil
+from pathlib import Path
+from cadgenbench.eval.evaluate import evaluate_result
+cand, gt = Path(sys.argv[1]), Path(sys.argv[2])
+with tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:
+    rd, gd = Path(rd), Path(gd)
+    (rd / 'result.json').write_text('{}')
+    shutil.copy(gt, gd / 'ground_truth.step')
+    try:
+        evaluate_result(rd, gd, candidate_step=cand)
+        d = json.loads((rd / 'result.json').read_text())
+        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))
+    except Exception as e:
+        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))
+`.trim();
+/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is
+*  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */
+function fixtureTasks() {
+	const g = join(CGB_DIR, "tests/fixtures/geometry");
+	return [
+		{
+			id: "box-10x20x30",
+			prompt: "A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).",
+			gtStep: join(g, "box_10_20_30.step")
+		},
+		{
+			id: "cube-10",
+			prompt: "A cube, 10 units on every side.",
+			gtStep: join(g, "box_10_10_10.step")
+		},
+		{
+			id: "sphere-10",
+			prompt: "A sphere of radius 10 units, centered at the origin.",
+			gtStep: join(g, "sphere_10.step")
+		}
+	];
+}
+function createCadGenBenchAdapter() {
+	return {
+		name: "cadgenbench",
+		async preflight() {
+			const r = await execFileAsync(CGB_VENV_PY, ["-c", "import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")"], { timeout: 6e4 }).catch((e) => ({
+				stdout: "",
+				stderr: e instanceof Error ? e.message : String(e)
+			}));
+			if (!/ok/.test(r.stdout)) throw new Error(`cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\nFix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`);
+		},
+		async loadTasks(opts = {}) {
+			let tasks = fixtureTasks();
+			const hard = process.env.CGB_HARD_DIR;
+			if (hard) {
+				const { readFile } = await import("node:fs/promises");
+				tasks = JSON.parse(await readFile(join(hard, "tasks.json"), "utf8"));
+			}
+			if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
+			if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
+			const meta = (gtStep) => ({
+				gtStep,
+				resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? .9)
+			});
+			return tasks.map((t) => ({
+				id: t.id,
+				prompt: t.prompt,
+				metadata: meta(t.gtStep)
+			}));
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			const { gtStep, resolveThreshold } = task.metadata;
+			if (!artifact.includes("ISO-10303-21")) return {
+				resolved: false,
+				score: 0,
+				detail: "artifact is not a STEP file"
+			};
+			const dir = await mkdtemp(join(tmpdir(), "cgb-judge-"));
+			const cand = join(dir, "candidate.step");
+			const scorer = join(dir, "score.py");
+			try {
+				await writeFile(cand, artifact);
+				await writeFile(scorer, SCORE_PY);
+				const r = await execFileAsync("xvfb-run", [
+					"-a",
+					CGB_VENV_PY,
+					scorer,
+					cand,
+					gtStep
+				], {
+					maxBuffer: 1 << 26,
+					timeout: 18e4
+				}).catch((e) => ({
+					stdout: e.stdout ?? "",
+					stderr: e instanceof Error ? e.message : String(e)
+				}));
+				const m = /CGB_SCORE (\{.*\})/.exec(r.stdout);
+				if (!m) return {
+					resolved: false,
+					score: 0,
+					detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}`
+				};
+				const v = JSON.parse(m[1]);
+				const score = typeof v.cad_score === "number" ? v.cad_score : 0;
+				return {
+					resolved: score >= resolveThreshold,
+					score,
+					detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ""}`
+				};
+			} finally {
+				await rm(dir, {
+					recursive: true,
+					force: true
+				}).catch(() => {});
+			}
+		}
+	};
+}
+//#endregion
+export { createCadGenBenchAdapter as t };
+
+//# sourceMappingURL=cadgenbench-DRhczfsG.js.map

package/dist/cadgenbench-DRhczfsG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadgenbench-DRhczfsG.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const { content, usage: u } = await routerChatWithUsage(cfg, [\n        { role: 'system', content: sys },\n        { role: 'user', content: user },\n      ])\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}