@tangle-network/agent-bench 0.8.12 → 0.8.16

package/dist/benchmarks/appworld.js.map

@@ -1 +1 @@
-{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  type OutputAdapter,\n  streamAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * Runtime's profile-bound `router-tools` executor with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\nexport interface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  cost_usd?: number\n  transcript?: string\n}\n\ninterface ReactRuntimeUsage {\n  input: number\n  output: number\n  costUsd?: number\n  tokensKnown?: boolean\n  usdKnown?: boolean\n}\n\n/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.\n * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without\n * discarding the episode's task evidence. */\nexport function appworldReactResultWithUsage(\n  verdict: ReactResult,\n  usage: ReactRuntimeUsage,\n  turns: number | undefined,\n  transcript: string,\n): ReactResult {\n  return {\n    ...verdict,\n    ...(turns !== undefined ? { turns } : {}),\n    ...(usage.tokensKnown === false\n      ? {}\n      : { input_tokens: usage.input, output_tokens: usage.output }),\n    ...(usage.usdKnown === false || usage.costUsd === undefined\n      ? {}\n      : { cost_usd: usage.costUsd }),\n    transcript,\n  }\n}\n\n/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */\nexport function appworldReactUsageEvent(\n  result: ReactResult,\n  model: string,\n): { type: 'llm_call'; data: Record<string, unknown> } | undefined {\n  const hasTokens =\n    typeof result.input_tokens === 'number' && typeof result.output_tokens === 'number'\n  const hasCost = typeof result.cost_usd === 'number'\n  if (!hasTokens && !hasCost) return undefined\n  return {\n    type: 'llm_call',\n    data: {\n      model,\n      ...(hasTokens\n        ? { tokensIn: result.input_tokens, tokensOut: result.output_tokens }\n        : {}),\n      ...(hasCost ? { costUsd: result.cost_usd } : {}),\n    },\n  }\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  signal: AbortSignal,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  signal.throwIfAborted()\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const stopChild = (): void => {\n    if (!child.killed) child.kill('SIGTERM')\n  }\n  signal.addEventListener('abort', stopChild, { once: true })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      if (signal.aborted) {\n        reject(signal.reason)\n        return\n      }\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        signal.removeEventListener('abort', onAbort)\n        reject(\n          signal.aborted\n            ? signal.reason\n            : new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`),\n        )\n      }\n      const onAbort = (): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        const index = pending.indexOf(onLine)\n        if (index >= 0) pending.splice(index, 1)\n        reject(signal.reason)\n      }\n      const onLine = (line: string): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        signal.removeEventListener('abort', onAbort)\n        resolve(line)\n      }\n      pending.push(onLine)\n      child.once('exit', onExit)\n      signal.addEventListener('abort', onAbort, { once: true })\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    signal.removeEventListener('abort', stopChild)\n    child.stdin.end()\n    stopChild()\n  }\n}\n\ntype AppWorldWorldSession = typeof withWorldSession\ntype AppWorldComplete = (\n  body: Record<string, unknown>,\n  request?: {\n    readonly headers: Readonly<Record<string, string>>\n    readonly signal?: AbortSignal\n  },\n) => Promise<unknown>\n\n/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n  /** Offline-test seam; production always uses the Python AppWorld session above. */\n  runWorldSession?: AppWorldWorldSession\n  /** Offline-test seam; production uses Runtime's Router HTTP transport. */\n  complete?: AppWorldComplete\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  const runWorldSession = cfg.runWorldSession ?? withWorldSession\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string, promptOpts?: { signal?: AbortSignal }) {\n          const signal = promptOpts?.signal ?? new AbortController().signal\n          signal.throwIfAborted()\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await runWorldSession(taskId as string, split as string, signal, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const transcriptSteps: Array<{ args: string; result: string }> = []\n            const profile = {\n              name: 'appworld-react-worker',\n              harness: 'cli-base' as const,\n              model: {\n                provider: 'tangle-router',\n                default: cfg.model,\n                metadata: { maxTurns },\n              },\n              prompt: { systemPrompt: system },\n              tools: { execute_python: true },\n            }\n            const factory = createExecutor({\n              backend: 'router-tools',\n              routerBaseUrl: cfg.routerBaseUrl,\n              routerKey: cfg.routerKey,\n              ...(cfg.complete ? { complete: cfg.complete } : {}),\n              tools: [EXECUTE_TOOL],\n              executeToolCall: async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                const result = `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n                transcriptSteps.push({ args: JSON.stringify(args), result })\n                return result\n              },\n            })\n            const loop = await collectAgentTurn(\n              streamAgentTurn(\n                { kind: 'executor', factory, profile },\n                `Task: ${instruction}`,\n                { signal },\n              ),\n            )\n            if (loop.status !== 'completed') {\n              throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`)\n            }\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = transcriptSteps\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            const finalEvent = loop.events.at(-1)\n            const resultMetadata =\n              finalEvent?.type === 'final' && finalEvent.metadata?.result\n                ? (finalEvent.metadata.result as { spent?: { iterations?: number } })\n                : undefined\n            return appworldReactResultWithUsage(\n              verdict,\n              loop.usage,\n              resultMetadata?.spent?.iterations,\n              transcript,\n            )\n          })\n          const usageEvent = appworldReactUsageEvent(out, cfg.model)\n          if (usageEvent) yield usageEvent\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;;;;AA+CA,SAAgB,6BACd,SACA,OACA,OACA,YACa;CACb,OAAO;EACL,GAAG;EACH,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,MAAM,gBAAgB,QACtB,CAAC,IACD;GAAE,cAAc,MAAM;GAAO,eAAe,MAAM;EAAO;EAC7D,GAAI,MAAM,aAAa,SAAS,MAAM,YAAY,KAAA,IAC9C,CAAC,IACD,EAAE,UAAU,MAAM,QAAQ;EAC9B;CACF;AACF;;AAGA,SAAgB,wBACd,QACA,OACiE;CACjE,MAAM,YACJ,OAAO,OAAO,iBAAiB,YAAY,OAAO,OAAO,kBAAkB;CAC7E,MAAM,UAAU,OAAO,OAAO,aAAa;CAC3C,IAAI,CAAC,aAAa,CAAC,SAAS,OAAO,KAAA;CACnC,OAAO;EACL,MAAM;EACN,MAAM;GACJ;GACA,GAAI,YACA;IAAE,UAAU,OAAO;IAAc,WAAW,OAAO;GAAc,IACjE,CAAC;GACL,GAAI,UAAU,EAAE,SAAS,OAAO,SAAS,IAAI,CAAC;EAChD;CACF;AACF;AAEA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,QACA,IACY;CACZ,OAAO,eAAe;CACtB,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,kBAAwB;EAC5B,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,SAAS;CACzC;CACA,OAAO,iBAAiB,SAAS,WAAW,EAAE,MAAM,KAAK,CAAC;CAC1D,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,IAAI,OAAO,SAAS;GAClB,OAAO,OAAO,MAAM;GACpB;EACF;EACA,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,OAAO,oBAAoB,SAAS,OAAO;GAC3C,OACE,OAAO,UACH,OAAO,yBACP,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAClF;EACF;EACA,MAAM,gBAAsB;GAC1B,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,MAAM,QAAQ,QAAQ,QAAQ,MAAM;GACpC,IAAI,SAAS,GAAG,QAAQ,OAAO,OAAO,CAAC;GACvC,OAAO,OAAO,MAAM;EACtB;EACA,MAAM,UAAU,SAAuB;GACrC,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,OAAO,oBAAoB,SAAS,OAAO;GAC3C,QAAQ,IAAI;EACd;EACA,QAAQ,KAAK,MAAM;EACnB,MAAM,KAAK,QAAQ,MAAM;EACzB,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC1D,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,OAAO,oBAAoB,SAAS,SAAS;EAC7C,MAAM,MAAM,IAAI;EAChB,UAAU;CACZ;AACF;;AAYA,SAAgB,uBAAuB,KAS3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,MAAM,kBAAkB,IAAI,mBAAmB;CAC/C,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB,YAAuC;IACzE,MAAM,SAAS,YAAY,UAAU,IAAI,gBAAgB,CAAC,CAAC;IAC3D,OAAO,eAAe;IACtB,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,gBAAgB,QAAkB,OAAiB,QAAQ,OAAO,MAAM,gBAAgB;KACxG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,kBAA2D,CAAC;KAClE,MAAM,UAAU;MACd,MAAM;MACN,SAAS;MACT,OAAO;OACL,UAAU;OACV,SAAS,IAAI;OACb,UAAU,EAAE,SAAS;MACvB;MACA,QAAQ,EAAE,cAAc,OAAO;MAC/B,OAAO,EAAE,gBAAgB,KAAK;KAChC;KAgBA,MAAM,OAAO,MAAM,iBACjB,gBACE;MAAE,MAAM;MAAY,SAjBR,eAAe;OAC7B,SAAS;OACT,eAAe,IAAI;OACnB,WAAW,IAAI;OACf,GAAI,IAAI,WAAW,EAAE,UAAU,IAAI,SAAS,IAAI,CAAC;OACjD,OAAO,CAAC,YAAY;OACpB,iBAAiB,OAAO,MAAM,SAAS;QACrC,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;QAC7D,MAAM,MAAM,MAAM,KAAK;SAAE,IAAI;SAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;QAAE,CAAC;QACvE,MAAM,OAAO,IAAI,mBAAmB;QACpC,MAAM,SAAS,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;QAC/I,gBAAgB,KAAK;SAAE,MAAM,KAAK,UAAU,IAAI;SAAG;QAAO,CAAC;QAC3D,OAAO;OACT;MACF,CAG8B;MAAG;KAAQ,GACrC,SAAS,eACT,EAAE,OAAO,CACX,CACF;KACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,4BAA4B,KAAK,QAAQ;KAElF,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,gBAChB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,MAAM,aAAa,KAAK,OAAO,GAAG,EAAE;KACpC,MAAM,iBACJ,YAAY,SAAS,WAAW,WAAW,UAAU,SAChD,WAAW,SAAS,SACrB,KAAA;KACN,OAAO,6BACL,SACA,KAAK,OACL,gBAAgB,OAAO,YACvB,UACF;IACF,CAAC;IACD,MAAM,aAAa,wBAAwB,KAAK,IAAI,KAAK;IACzD,IAAI,YAAY,MAAM;IACtB,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}
+{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  type OutputAdapter,\n  streamAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * Runtime's profile-bound `router-tools` executor with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\nexport interface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  cost_usd?: number\n  transcript?: string\n}\n\ninterface ReactRuntimeUsage {\n  input: number\n  output: number\n  costUsd?: number\n  tokensKnown?: boolean\n  usdKnown?: boolean\n}\n\n/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.\n * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without\n * discarding the episode's task evidence. */\nexport function appworldReactResultWithUsage(\n  verdict: ReactResult,\n  usage: ReactRuntimeUsage,\n  turns: number | undefined,\n  transcript: string,\n): ReactResult {\n  return {\n    ...verdict,\n    ...(turns !== undefined ? { turns } : {}),\n    ...(usage.tokensKnown === false\n      ? {}\n      : { input_tokens: usage.input, output_tokens: usage.output }),\n    ...(usage.usdKnown === false || usage.costUsd === undefined\n      ? {}\n      : { cost_usd: usage.costUsd }),\n    transcript,\n  }\n}\n\n/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */\nexport function appworldReactUsageEvent(\n  result: ReactResult,\n  model: string,\n): { type: 'llm_call'; data: Record<string, unknown> } | undefined {\n  const hasTokens =\n    typeof result.input_tokens === 'number' && typeof result.output_tokens === 'number'\n  const hasCost = typeof result.cost_usd === 'number'\n  if (!hasTokens && !hasCost) return undefined\n  return {\n    type: 'llm_call',\n    data: {\n      model,\n      ...(hasTokens\n        ? { tokensIn: result.input_tokens, tokensOut: result.output_tokens }\n        : {}),\n      ...(hasCost ? { costUsd: result.cost_usd } : {}),\n    },\n  }\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  signal: AbortSignal,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  signal.throwIfAborted()\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const stopChild = (): void => {\n    if (!child.killed) child.kill('SIGTERM')\n  }\n  signal.addEventListener('abort', stopChild, { once: true })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      if (signal.aborted) {\n        reject(signal.reason)\n        return\n      }\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        signal.removeEventListener('abort', onAbort)\n        reject(\n          signal.aborted\n            ? signal.reason\n            : new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`),\n        )\n      }\n      const onAbort = (): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        const index = pending.indexOf(onLine)\n        if (index >= 0) pending.splice(index, 1)\n        reject(signal.reason)\n      }\n      const onLine = (line: string): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        signal.removeEventListener('abort', onAbort)\n        resolve(line)\n      }\n      pending.push(onLine)\n      child.once('exit', onExit)\n      signal.addEventListener('abort', onAbort, { once: true })\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    signal.removeEventListener('abort', stopChild)\n    child.stdin.end()\n    stopChild()\n  }\n}\n\ntype AppWorldWorldSession = typeof withWorldSession\ntype AppWorldComplete = (\n  body: Record<string, unknown>,\n  request?: {\n    readonly headers: Readonly<Record<string, string>>\n    readonly signal?: AbortSignal\n  },\n) => Promise<unknown>\n\n/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n  /** Offline-test seam; production always uses the Python AppWorld session above. */\n  runWorldSession?: AppWorldWorldSession\n  /** Offline-test seam; production uses Runtime's Router HTTP transport. */\n  complete?: AppWorldComplete\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  const runWorldSession = cfg.runWorldSession ?? withWorldSession\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string, promptOpts?: { signal?: AbortSignal }) {\n          const signal = promptOpts?.signal ?? new AbortController().signal\n          signal.throwIfAborted()\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await runWorldSession(taskId as string, split as string, signal, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const transcriptSteps: Array<{ args: string; result: string }> = []\n            const profile = {\n              name: 'appworld-react-worker',\n              harness: 'cli-base' as const,\n              model: {\n                provider: 'tangle-router',\n                default: cfg.model,\n                metadata: { maxTurns },\n              },\n              prompt: { systemPrompt: system },\n              tools: { execute_python: true },\n            }\n            const factory = createExecutor({\n              backend: 'router-tools',\n              routerBaseUrl: cfg.routerBaseUrl,\n              routerKey: cfg.routerKey,\n              ...(cfg.complete ? { complete: cfg.complete } : {}),\n              tools: [EXECUTE_TOOL],\n              executeToolCall: async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                const result = `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n                transcriptSteps.push({ args: JSON.stringify(args), result })\n                return result\n              },\n            })\n            const loop = await collectAgentTurn(\n              streamAgentTurn(\n                { kind: 'executor', factory, profile },\n                { prompt: `Task: ${instruction}` },\n                { signal },\n              ),\n            )\n            if (loop.status !== 'completed') {\n              throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`)\n            }\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = transcriptSteps\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            const finalEvent = loop.events.at(-1)\n            const resultMetadata =\n              finalEvent?.type === 'final' && finalEvent.metadata?.result\n                ? (finalEvent.metadata.result as { spent?: { iterations?: number } })\n                : undefined\n            return appworldReactResultWithUsage(\n              verdict,\n              loop.usage,\n              resultMetadata?.spent?.iterations,\n              transcript,\n            )\n          })\n          const usageEvent = appworldReactUsageEvent(out, cfg.model)\n          if (usageEvent) yield usageEvent\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;;;;AA+CA,SAAgB,6BACd,SACA,OACA,OACA,YACa;CACb,OAAO;EACL,GAAG;EACH,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,MAAM,gBAAgB,QACtB,CAAC,IACD;GAAE,cAAc,MAAM;GAAO,eAAe,MAAM;EAAO;EAC7D,GAAI,MAAM,aAAa,SAAS,MAAM,YAAY,KAAA,IAC9C,CAAC,IACD,EAAE,UAAU,MAAM,QAAQ;EAC9B;CACF;AACF;;AAGA,SAAgB,wBACd,QACA,OACiE;CACjE,MAAM,YACJ,OAAO,OAAO,iBAAiB,YAAY,OAAO,OAAO,kBAAkB;CAC7E,MAAM,UAAU,OAAO,OAAO,aAAa;CAC3C,IAAI,CAAC,aAAa,CAAC,SAAS,OAAO,KAAA;CACnC,OAAO;EACL,MAAM;EACN,MAAM;GACJ;GACA,GAAI,YACA;IAAE,UAAU,OAAO;IAAc,WAAW,OAAO;GAAc,IACjE,CAAC;GACL,GAAI,UAAU,EAAE,SAAS,OAAO,SAAS,IAAI,CAAC;EAChD;CACF;AACF;AAEA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,QACA,IACY;CACZ,OAAO,eAAe;CACtB,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,kBAAwB;EAC5B,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,SAAS;CACzC;CACA,OAAO,iBAAiB,SAAS,WAAW,EAAE,MAAM,KAAK,CAAC;CAC1D,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,IAAI,OAAO,SAAS;GAClB,OAAO,OAAO,MAAM;GACpB;EACF;EACA,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,OAAO,oBAAoB,SAAS,OAAO;GAC3C,OACE,OAAO,UACH,OAAO,yBACP,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAClF;EACF;EACA,MAAM,gBAAsB;GAC1B,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,MAAM,QAAQ,QAAQ,QAAQ,MAAM;GACpC,IAAI,SAAS,GAAG,QAAQ,OAAO,OAAO,CAAC;GACvC,OAAO,OAAO,MAAM;EACtB;EACA,MAAM,UAAU,SAAuB;GACrC,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,OAAO,oBAAoB,SAAS,OAAO;GAC3C,QAAQ,IAAI;EACd;EACA,QAAQ,KAAK,MAAM;EACnB,MAAM,KAAK,QAAQ,MAAM;EACzB,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC1D,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,OAAO,oBAAoB,SAAS,SAAS;EAC7C,MAAM,MAAM,IAAI;EAChB,UAAU;CACZ;AACF;;AAYA,SAAgB,uBAAuB,KAS3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,MAAM,kBAAkB,IAAI,mBAAmB;CAC/C,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB,YAAuC;IACzE,MAAM,SAAS,YAAY,UAAU,IAAI,gBAAgB,CAAC,CAAC;IAC3D,OAAO,eAAe;IACtB,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,gBAAgB,QAAkB,OAAiB,QAAQ,OAAO,MAAM,gBAAgB;KACxG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,kBAA2D,CAAC;KAClE,MAAM,UAAU;MACd,MAAM;MACN,SAAS;MACT,OAAO;OACL,UAAU;OACV,SAAS,IAAI;OACb,UAAU,EAAE,SAAS;MACvB;MACA,QAAQ,EAAE,cAAc,OAAO;MAC/B,OAAO,EAAE,gBAAgB,KAAK;KAChC;KAgBA,MAAM,OAAO,MAAM,iBACjB,gBACE;MAAE,MAAM;MAAY,SAjBR,eAAe;OAC7B,SAAS;OACT,eAAe,IAAI;OACnB,WAAW,IAAI;OACf,GAAI,IAAI,WAAW,EAAE,UAAU,IAAI,SAAS,IAAI,CAAC;OACjD,OAAO,CAAC,YAAY;OACpB,iBAAiB,OAAO,MAAM,SAAS;QACrC,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;QAC7D,MAAM,MAAM,MAAM,KAAK;SAAE,IAAI;SAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;QAAE,CAAC;QACvE,MAAM,OAAO,IAAI,mBAAmB;QACpC,MAAM,SAAS,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;QAC/I,gBAAgB,KAAK;SAAE,MAAM,KAAK,UAAU,IAAI;SAAG;QAAO,CAAC;QAC3D,OAAO;OACT;MACF,CAG8B;MAAG;KAAQ,GACrC,EAAE,QAAQ,SAAS,cAAc,GACjC,EAAE,OAAO,CACX,CACF;KACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,4BAA4B,KAAK,QAAQ;KAElF,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,gBAChB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,MAAM,aAAa,KAAK,OAAO,GAAG,EAAE;KACpC,MAAM,iBACJ,YAAY,SAAS,WAAW,WAAW,UAAU,SAChD,WAAW,SAAS,SACrB,KAAA;KACN,OAAO,6BACL,SACA,KAAK,OACL,gBAAgB,OAAO,YACvB,UACF;IACF,CAAC;IACD,MAAM,aAAa,wBAAwB,KAAK,IAAI,KAAK;IACzD,IAAI,YAAY,MAAM;IACtB,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}

package/dist/benchmarks/cadbench.js

@@ -1,2 +1,2 @@
-import { t as createCadBenchAdapter } from "../cadbench-BLSyxR1N.js";
+import { t as createCadBenchAdapter } from "../cadbench-BRF-59Mt.js";
 export { createCadBenchAdapter };

package/dist/benchmarks/cadgenbench.js

@@ -1,2 +1,2 @@
-import { t as createCadGenBenchAdapter } from "../cadgenbench-x2OFkf8y.js";
+import { t as createCadGenBenchAdapter } from "../cadgenbench-DXtGkuW3.js";
 export { createCadGenBenchAdapter };

package/dist/benchmarks/finresearchbench.js

@@ -1,5 +1,5 @@
 import { benchRoot } from "./_harness.js";
-import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "../router-turn-uTYO6KQ1.js";
 import { readFile, stat } from "node:fs/promises";
 import { join } from "node:path";
 //#region src/benchmarks/finresearchbench.ts

package/dist/benchmarks/finsearchcomp.js

@@ -1,4 +1,4 @@
-import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "../router-turn-uTYO6KQ1.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";

package/dist/benchmarks/frames.js

@@ -1,4 +1,4 @@
-import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "../router-turn-uTYO6KQ1.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";

package/dist/benchmarks/simpleqa.js

@@ -1,4 +1,4 @@
-import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "../router-turn-uTYO6KQ1.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";

package/dist/benchmarks/trata-hedge.js

@@ -1,4 +1,4 @@
-import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "../router-turn-uTYO6KQ1.js";
 import { join } from "node:path";
 import { readFileSync, readdirSync, statSync } from "node:fs";
 //#region src/benchmarks/trata-hedge.ts

package/dist/{cadbench-BLSyxR1N.js → cadbench-BRF-59Mt.js}

@@ -1,4 +1,4 @@
-import { t as runBenchRouterTurn } from "./router-turn-C2wMiDoo.js";
+import { t as runBenchRouterTurn } from "./router-turn-uTYO6KQ1.js";
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
@@ -283,4 +283,4 @@ function createCadBenchAdapter() {
 //#endregion
 export { createCadBenchAdapter as t };
 
-//# sourceMappingURL=cadbench-BLSyxR1N.js.map
+//# sourceMappingURL=cadbench-BRF-59Mt.js.map

package/dist/{cadbench-BLSyxR1N.js.map → cadbench-BRF-59Mt.js.map}

@@ -1 +1 @@
-{"version":3,"file":"cadbench-BLSyxR1N.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'blender-worker',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: cfg.model,\n              metadata: { temperature: 0.3 },\n            },\n            prompt: { systemPrompt: directive },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\nimport { runBenchRouterTurn } from '../router-turn'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: base,\n      routerKey: key,\n      profile: {\n        name: 'cadbench-vision-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: model,\n          metadata: {\n            temperature: 0,\n            maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500),\n          },\n        },\n      },\n    },\n    { messages: [{ role: 'user', content }] },\n  )\n  const raw = turn.finalText\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;ACzFA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CAoBjF,MAAM,OAAM,MAnBO,mBACjB;EACE,eAAe;EACf,WAAW;EACX,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS;IACT,UAAU;KACR,aAAa;KACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,IAAI;IACxD;GACF;EACF;CACF,GACA,EAAE,UAAU,CAAC;EAAE,MAAM;EAAQ;CAAQ,CAAC,EAAE,CAC1C,EAAA,CACiB;CACjB,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}
+{"version":3,"file":"cadbench-BRF-59Mt.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'blender-worker',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: cfg.model,\n              metadata: { temperature: 0.3 },\n            },\n            prompt: { systemPrompt: directive },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\nimport { runBenchRouterTurn } from '../router-turn'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: base,\n      routerKey: key,\n      profile: {\n        name: 'cadbench-vision-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: model,\n          metadata: {\n            temperature: 0,\n            maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500),\n          },\n        },\n      },\n    },\n    { messages: [{ role: 'user', content }] },\n  )\n  const raw = turn.finalText\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;ACzFA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CAoBjF,MAAM,OAAM,MAnBO,mBACjB;EACE,eAAe;EACf,WAAW;EACX,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS;IACT,UAAU;KACR,aAAa;KACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,IAAI;IACxD;GACF;EACF;CACF,GACA,EAAE,UAAU,CAAC;EAAE,MAAM;EAAQ;CAAQ,CAAC,EAAE,CAC1C,EAAA,CACiB;CACjB,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}

package/dist/{cadgenbench-x2OFkf8y.js → cadgenbench-DXtGkuW3.js}

@@ -1,4 +1,4 @@
-import "./router-turn-C2wMiDoo.js";
+import "./router-turn-uTYO6KQ1.js";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
@@ -148,4 +148,4 @@ function createCadGenBenchAdapter() {
 //#endregion
 export { createCadGenBenchAdapter as t };
 
-//# sourceMappingURL=cadgenbench-x2OFkf8y.js.map
+//# sourceMappingURL=cadgenbench-DXtGkuW3.js.map

package/dist/{cadgenbench-x2OFkf8y.js.map → cadgenbench-DXtGkuW3.js.map}

@@ -1 +1 @@
-{"version":3,"file":"cadgenbench-x2OFkf8y.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'build123d-worker',\n            model: { provider: 'tangle-router', default: cfg.model },\n            prompt: { systemPrompt: sys },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}
+{"version":3,"file":"cadgenbench-DXtGkuW3.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'build123d-worker',\n            model: { provider: 'tangle-router', default: cfg.model },\n            prompt: { systemPrompt: sys },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}

package/dist/index.js

@@ -265,14 +265,19 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 		...timeoutMs ? { timeoutMs } : {}
 	});
 	const harness = cell.harness ?? cell.profile?.metadata?.backendType ?? "opencode";
-	const profile = cell.profile ?? {
-		name: cell.label,
+	const profileProvider = cell.profile?.model?.provider ?? "tangle-router";
+	const profile = {
+		...cell.profile ?? { name: cell.label },
 		harness,
 		model: {
-			provider: "tangle-router",
+			...cell.profile?.model,
+			provider: profileProvider,
 			default: cell.model
 		},
-		metadata: { backendType: harness }
+		metadata: {
+			...cell.profile?.metadata,
+			backendType: harness
+		}
 	};
 	const uniq = Math.random().toString(36).slice(2, 8);
 	const agentRun = {
@@ -285,7 +290,7 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 			backend: {
 				type: harness,
 				model: {
-					provider: "openai",
+					provider: profileProvider,
 					model: cell.model,
 					baseUrl: routerBaseUrl
 				}