@tangle-network/agent-bench 0.7.1 → 0.8.1

package/dist/cadgenbench-DF7hYHdl.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cadgenbench-DF7hYHdl.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const { content, usage: u } = await routerChatWithUsage(cfg, [\n        { role: 'system', content: sys },\n        { role: 'user', content: user },\n      ])\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}

package/scripts/trata-hedge/solve.py

@@ -1,83 +0,0 @@
-"""trata-hedge-bench solver — our system's analyst, graded by THEIR Gemini judge.
-
-Reads a task's instruction.md + data corpus, writes a cited analysis to answer.txt.
-The corpus is large (~5MB/task), so a single-shot router call is a LOWER-BOUND
-baseline (it sees only what fits in context) — the bench is built for agentic
-exploration, which our sandbox runtime provides (see README; pending sandbox health).
-
-Notes learned the hard way:
-- router.tangle.tools is behind Cloudflare bot-fight: a default Python-urllib
-  User-Agent gets 403 (CF 1010) on large bodies → send a browser UA.
-- gpt-4.1 is NOT keyed for direct router calls (403); gpt-4o / claude-sonnet-4-6 /
-  gpt-4o-mini are. Very large bodies (~350KB) can 503 — keep DATA_BUDGET in range.
-
-  WORKER_MODEL=gpt-4o DATA_BUDGET=160000 python3 solve.py <env-dir> <out.txt>
-"""
-import json
-import os
-import sys
-import urllib.request
-from pathlib import Path
-
-env = Path(sys.argv[1])
-out = sys.argv[2] if len(sys.argv) > 2 else "/tmp/thb-answer.txt"
-model = os.environ.get("WORKER_MODEL", "gpt-4o")
-budget = int(os.environ.get("DATA_BUDGET", "160000"))
-instruction = (env / "instruction.md").read_text()
-data_dir = env / "environment" / "data"
-
-
-def rank(p: Path) -> int:
-    s = str(p)
-    return (0 if "earnings_call" in s else 1 if "financials" in s else 2 if "company_profiles" in s
-            else 3 if "press_releases" in s else 4)
-
-
-files = sorted((p for p in data_dir.rglob("*") if p.is_file()), key=rank)
-blocks, used, cited = [], 0, []
-for p in files:
-    rel = p.relative_to(data_dir)
-    try:
-        c = p.read_text(errors="replace")
-    except Exception:
-        continue
-    if used + len(c) > budget:
-        continue
-    used += len(c)
-    cited.append(str(rel))
-    blocks.append(f"\n=== FILE: data/{rel} ===\n{c}")
-print(f"[solve] {len(cited)}/{len(files)} files in context ({used} chars)", file=sys.stderr)
-
-prompt = (
-    instruction
-    + "\n\n--- AVAILABLE DATA (cite files by their `data/<path>` name inline) ---\n"
-    + "".join(blocks)
-    + "\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/<path>`."
-)
-
-key = os.environ["TANGLE_API_KEY"]
-base = os.environ.get("ROUTER_BASE", "https://router.tangle.tools/v1")
-body = json.dumps({
-    "model": model,
-    "messages": [{"role": "user", "content": prompt}],
-    "temperature": float(os.environ.get("TEMPERATURE", "0.5")),
-    "max_tokens": int(os.environ.get("MAX_TOKENS", "6000")),
-}).encode()
-req = urllib.request.Request(
-    f"{base}/chat/completions", data=body,
-    headers={
-        "Authorization": f"Bearer {key}",
-        "Content-Type": "application/json",
-        # browser UA: router is behind Cloudflare bot-fight, default urllib UA → 403 on big bodies
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
-    },
-)
-try:
-    with urllib.request.urlopen(req, timeout=300) as r:
-        resp = json.load(r)
-except urllib.error.HTTPError as e:
-    print(f"[solve] HTTP {e.code}: {e.read().decode(errors='replace')[:400]}", file=sys.stderr)
-    raise
-answer = resp["choices"][0]["message"]["content"]
-Path(out).write_text(answer)
-print(f"[solve] wrote {len(answer)} chars -> {out} (model={model})", file=sys.stderr)