@tangle-network/agent-bench 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/HARNESS.md +1 -1
  3. package/dist/adapters.js +2 -2
  4. package/dist/benchmarks/appworld.d.ts +61 -2
  5. package/dist/benchmarks/appworld.js +111 -40
  6. package/dist/benchmarks/appworld.js.map +1 -1
  7. package/dist/benchmarks/cadbench.js +1 -1
  8. package/dist/benchmarks/cadgenbench.js +1 -1
  9. package/dist/benchmarks/finresearchbench.js +16 -22
  10. package/dist/benchmarks/finresearchbench.js.map +1 -1
  11. package/dist/benchmarks/finsearchcomp.js +16 -22
  12. package/dist/benchmarks/finsearchcomp.js.map +1 -1
  13. package/dist/benchmarks/frames.js +18 -20
  14. package/dist/benchmarks/frames.js.map +1 -1
  15. package/dist/benchmarks/simpleqa.js +15 -19
  16. package/dist/benchmarks/simpleqa.js.map +1 -1
  17. package/dist/benchmarks/trata-hedge.js +20 -23
  18. package/dist/benchmarks/trata-hedge.js.map +1 -1
  19. package/dist/{cadbench-BrpwOU6A.js → cadbench-BLSyxR1N.js} +21 -20
  20. package/dist/cadbench-BLSyxR1N.js.map +1 -0
  21. package/dist/{cadgenbench-DF7hYHdl.js → cadgenbench-x2OFkf8y.js} +2 -2
  22. package/dist/cadgenbench-x2OFkf8y.js.map +1 -0
  23. package/dist/index.d.ts +0 -1
  24. package/dist/index.js +10 -9
  25. package/dist/index.js.map +1 -1
  26. package/dist/router-turn-C2wMiDoo.js +31 -0
  27. package/dist/router-turn-C2wMiDoo.js.map +1 -0
  28. package/package.json +7 -7
  29. package/scripts/appworld_driver.py +1 -170
  30. package/scripts/run-package-tests.mjs +56 -36
  31. package/scripts/run-package-tests.test.mjs +33 -0
  32. package/scripts/trata-hedge/README.md +3 -3
  33. package/scripts/trata-hedge/run.sh +6 -2
  34. package/src/aec-gate.mts +30 -9
  35. package/src/agent-graphs-gen2.mts +523 -0
  36. package/src/agent-graphs-gen3.mts +660 -0
  37. package/src/agent-graphs-improve/offline-seams.mts +128 -0
  38. package/src/agent-graphs-improve.mts +747 -0
  39. package/src/atom-humaneval.mts +76 -55
  40. package/src/atom-mcp-e2e.mts +27 -13
  41. package/src/benchmarks/appworld.test.mts +119 -1
  42. package/src/benchmarks/appworld.ts +156 -33
  43. package/src/benchmarks/cadbench.ts +21 -8
  44. package/src/benchmarks/finresearchbench.ts +20 -16
  45. package/src/benchmarks/finsearchcomp.ts +20 -20
  46. package/src/benchmarks/frames.ts +19 -18
  47. package/src/benchmarks/simpleqa.ts +19 -17
  48. package/src/benchmarks/trata-hedge.ts +26 -18
  49. package/src/browser/ui-reviewer.ts +1 -1
  50. package/src/clbench-context-gate.mts +45 -9
  51. package/src/cloud-loop.mts +24 -6
  52. package/src/commit0-env-run.mts +6 -3
  53. package/src/commit0-env.ts +2 -3
  54. package/src/commit0-gate.mts +53 -24
  55. package/src/corpus.ts +3 -3
  56. package/src/david-attribution.mts +28 -9
  57. package/src/david-goliath.mts +31 -12
  58. package/src/egress-probe.mts +3 -3
  59. package/src/examples/lean-proof-gate.mts +10 -3
  60. package/src/examples/math-demo.mts +10 -3
  61. package/src/examples/strategy-demo.mts +10 -4
  62. package/src/fleet.mts +30 -8
  63. package/src/gate-cli.mts +9 -4
  64. package/src/gate.test.mts +7 -2
  65. package/src/gate.ts +23 -75
  66. package/src/generate-eval/certify.ts +13 -5
  67. package/src/hev-eval.mts +39 -10
  68. package/src/hev-improve.mts +28 -16
  69. package/src/hev-structural.mts +28 -20
  70. package/src/humaneval-gate.mts +24 -8
  71. package/src/humaneval-object-ablation.mts +52 -14
  72. package/src/humaneval-repair-gate.mts +53 -14
  73. package/src/mbpp-structural.mts +28 -20
  74. package/src/mcp-mount-probe.mts +23 -11
  75. package/src/official-optimizer-config.mts +61 -12
  76. package/src/official-optimizer-config.test.mts +72 -3
  77. package/src/quant-arena/profiles/quant-researcher.profile.json +6 -1
  78. package/src/quant-arena/quant-loop.mts +88 -61
  79. package/src/research-shot.ts +21 -11
  80. package/src/resolve-client.ts +6 -8
  81. package/src/rollout-ledger/backfill-swe-arena.test.mts +7 -2
  82. package/src/router-executor.ts +1 -1
  83. package/src/router-turn.ts +135 -0
  84. package/src/run-benchmarks.ts +7 -2
  85. package/src/sandbox-run.ts +23 -41
  86. package/src/search-bench/bridge.ts +66 -8
  87. package/src/search-bench/parametric-check.mts +19 -4
  88. package/src/search-bench/profiles.ts +24 -9
  89. package/src/search-bench/run.mts +3 -4
  90. package/src/smoke-structural-rollout.mts +21 -5
  91. package/src/supervisor-arena.mts +28 -20
  92. package/src/swe-arena/arms.ts +73 -15
  93. package/src/swe-arena/capacity.ts +44 -41
  94. package/src/swe-arena/diagnosis-ensemble.ts +51 -48
  95. package/src/swe-arena/execution.test.mts +12 -12
  96. package/src/swe-arena/gepa-seat.mts +12 -5
  97. package/src/swe-arena/gepa-seat.test.mts +45 -33
  98. package/src/swe-arena/outer-loop.mts +94 -70
  99. package/src/swe-arena/premeasured-from-cells.mts +16 -1
  100. package/src/swe-arena/premeasured-from-cells.test.mts +21 -1
  101. package/src/swe-arena/profiles/deepseek-author.profile.json +12 -0
  102. package/src/swe-arena/profiles/default-author.profile.json +9 -1
  103. package/src/swe-arena/proposer-fanout.mts +52 -99
  104. package/src/swe-arena/proposer-fanout.test.mts +92 -57
  105. package/src/swe-arena/proposer-provenance.mts +1 -1
  106. package/src/swe-code-improve.mts +30 -52
  107. package/src/swe-emit-patch.mts +26 -5
  108. package/src/swe-improve.mts +27 -8
  109. package/src/swe-jail.test.ts +128 -0
  110. package/src/swe-jail.ts +106 -22
  111. package/src/swe-local-proof.mts +27 -5
  112. package/src/swe-repro-calibrate.mts +5 -0
  113. package/src/swe-self-improve.mts +45 -10
  114. package/src/swe-stream.mts +68 -6
  115. package/src/swe-structural.mts +46 -7
  116. package/src/tb-container-executor.mts +38 -6
  117. package/src/tb-container-executor.test.mts +11 -3
  118. package/src/tb-supervisor-sidecar.mts +11 -27
  119. package/src/terminal-compare.ts +38 -18
  120. package/src/trata-gate.mts +26 -26
  121. package/src/trata-gepa.mts +25 -16
  122. package/src/trata-hedge-solve.mts +80 -0
  123. package/src/worker-blender.ts +21 -11
  124. package/src/worker-browser.ts +18 -5
  125. package/src/worker-build123d.ts +17 -8
  126. package/src/worker-cad.ts +31 -12
  127. package/src/worker.ts +1 -1
  128. package/tb_agents/opencode_router_agent.py +84 -42
  129. package/dist/cadbench-BrpwOU6A.js.map +0 -1
  130. package/dist/cadgenbench-DF7hYHdl.js.map +0 -1
  131. package/scripts/trata-hedge/solve.py +0 -83
@@ -30,6 +30,7 @@
30
30
  import { appendFileSync } from 'node:fs'
31
31
  import { resolveAdapter } from './adapters'
32
32
  import type { BenchScore, BenchTask } from './benchmarks/types'
33
+ import { runBenchRouterTurn } from './router-turn'
33
34
  import { runPool } from './run-pool'
34
35
 
35
36
  function must(name: string): string {
@@ -61,33 +62,32 @@ async function workerComplete(
61
62
  cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
62
63
  ): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> {
63
64
  const startedAt = Date.now()
64
- const res = await fetch(`${cfg.routerBaseUrl}/chat/completions`, {
65
- method: 'POST',
66
- signal: AbortSignal.timeout(cfg.timeoutMs),
67
- headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` },
68
- body: JSON.stringify({
69
- model: cfg.model,
70
- temperature: 0,
71
- max_tokens: 4096,
72
- messages: [
73
- { role: 'system', content: ANALYST_SYSTEM },
74
- { role: 'user', content: task.prompt },
75
- ],
76
- }),
77
- })
78
- if (!res.ok) {
79
- const body = (await res.text()).slice(0, 300)
80
- throw new Error(`router ${res.status} for ${task.id}: ${body}`)
81
- }
82
- const j = (await res.json()) as {
83
- choices?: Array<{ message?: { content?: string } }>
84
- usage?: { prompt_tokens?: number; completion_tokens?: number }
85
- }
86
- const answer = j.choices?.[0]?.message?.content ?? ''
65
+ const result = await runBenchRouterTurn(
66
+ {
67
+ routerBaseUrl: cfg.routerBaseUrl,
68
+ routerKey: cfg.routerKey,
69
+ profile: {
70
+ name: 'trata-financial-analyst',
71
+ harness: 'cli-base',
72
+ model: {
73
+ provider: 'tangle-router',
74
+ default: cfg.model,
75
+ metadata: {
76
+ temperature: 0,
77
+ maxTokens: Number(process.env.WORKER_MAX_TOKENS ?? 4096),
78
+ },
79
+ },
80
+ prompt: { systemPrompt: ANALYST_SYSTEM },
81
+ },
82
+ timeoutMs: cfg.timeoutMs,
83
+ },
84
+ task.prompt,
85
+ )
86
+ if (result.usage.tokensKnown === false) throw new Error('worker provider omitted token usage')
87
87
  return {
88
- answer,
89
- inputTokens: j.usage?.prompt_tokens ?? 0,
90
- outputTokens: j.usage?.completion_tokens ?? 0,
88
+ answer: result.finalText,
89
+ inputTokens: result.usage.input,
90
+ outputTokens: result.usage.output,
91
91
  durationMs: Date.now() - startedAt,
92
92
  }
93
93
  }
@@ -60,6 +60,7 @@ import {
60
60
  officialOptimizerModel,
61
61
  requiredTokenPricing,
62
62
  } from './official-optimizer-config.mjs'
63
+ import { runBenchRouterTurn } from './router-turn'
63
64
 
64
65
  interface TrataScenario extends Scenario {
65
66
  task: BenchTask
@@ -122,23 +123,31 @@ async function chatComplete(
122
123
  messages: Array<{ role: string; content: string }>,
123
124
  signal: AbortSignal,
124
125
  ): Promise<{ content: string; usage?: { input: number; output: number } }> {
125
- const res = await fetch(`${baseUrl}/chat/completions`, {
126
- method: 'POST',
127
- signal: AbortSignal.any([signal, AbortSignal.timeout(180_000)]),
128
- headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
129
- body: JSON.stringify({ model, temperature: 0, max_tokens: maxTokens, messages }),
130
- })
131
- if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 300)}`)
132
- const j = (await res.json()) as {
133
- choices?: Array<{ message?: { content?: string } }>
134
- usage?: { prompt_tokens?: number; completion_tokens?: number }
126
+ const system = messages.find((message) => message.role === 'system')?.content
127
+ const result = await runBenchRouterTurn(
128
+ {
129
+ routerBaseUrl: baseUrl,
130
+ routerKey: key,
131
+ profile: {
132
+ name: 'trata-gepa-worker',
133
+ harness: 'cli-base',
134
+ model: {
135
+ provider: 'tangle-router',
136
+ default: model,
137
+ metadata: { temperature: 0, maxTokens },
138
+ },
139
+ ...(system ? { prompt: { systemPrompt: system } } : {}),
140
+ },
141
+ signal,
142
+ },
143
+ { messages: messages.filter((message) => message.role !== 'system') },
144
+ )
145
+ return {
146
+ content: result.finalText,
147
+ ...(result.usage.tokensKnown === false
148
+ ? {}
149
+ : { usage: { input: result.usage.input, output: result.usage.output } }),
135
150
  }
136
- const content = j.choices?.[0]?.message?.content ?? ''
137
- const usage =
138
- j.usage?.prompt_tokens != null
139
- ? { input: j.usage.prompt_tokens, output: j.usage.completion_tokens ?? 0 }
140
- : undefined
141
- return { content, usage }
142
151
  }
143
152
 
144
153
  async function main(): Promise<void> {
@@ -0,0 +1,80 @@
1
+ /** Single-shot lower-bound solver for trata-hedge-bench. */
2
+ import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
3
+ import { join, relative, resolve } from 'node:path'
4
+ import { runBenchRouterTurn } from './router-turn'
5
+
6
+ const environment = process.argv[2]
7
+ if (!environment) throw new Error('usage: solve.mts <env-dir> [out.txt]')
8
+ const output = process.argv[3] ?? '/tmp/thb-answer.txt'
9
+ const model = process.env.WORKER_MODEL
10
+ if (!model) throw new Error('WORKER_MODEL is required')
11
+ const dataBudget = Number(process.env.DATA_BUDGET ?? 160_000)
12
+ const maxTokens = Number(process.env.MAX_TOKENS ?? 6_000)
13
+ const temperature = Number(process.env.TEMPERATURE ?? 0.5)
14
+ const routerKey = process.env.TANGLE_API_KEY
15
+ if (!routerKey) throw new Error('TANGLE_API_KEY is required')
16
+ const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
17
+
18
+ for (const [name, value] of [
19
+ ['DATA_BUDGET', dataBudget],
20
+ ['MAX_TOKENS', maxTokens],
21
+ ] as const) {
22
+ if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`)
23
+ }
24
+ if (!Number.isFinite(temperature)) throw new Error('TEMPERATURE must be finite')
25
+
26
+ const root = resolve(environment)
27
+ const dataDir = join(root, 'environment', 'data')
28
+ const instruction = readFileSync(join(root, 'instruction.md'), 'utf8')
29
+
30
+ function filesUnder(dir: string): string[] {
31
+ return readdirSync(dir)
32
+ .flatMap((name) => {
33
+ const path = join(dir, name)
34
+ return statSync(path).isDirectory() ? filesUnder(path) : [path]
35
+ })
36
+ }
37
+
38
+ function rank(path: string): number {
39
+ if (path.includes('earnings_call')) return 0
40
+ if (path.includes('financials')) return 1
41
+ if (path.includes('company_profiles')) return 2
42
+ if (path.includes('press_releases')) return 3
43
+ return 4
44
+ }
45
+
46
+ let used = 0
47
+ const blocks: string[] = []
48
+ const files = filesUnder(dataDir).sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
49
+ for (const path of files) {
50
+ const content = readFileSync(path, 'utf8')
51
+ if (used + content.length > dataBudget) continue
52
+ used += content.length
53
+ blocks.push(`\n=== FILE: data/${relative(dataDir, path)} ===\n${content}`)
54
+ }
55
+ process.stderr.write(`[solve] ${blocks.length}/${files.length} files in context (${used} chars)\n`)
56
+
57
+ const prompt =
58
+ '--- AVAILABLE DATA (cite files by their `data/<path>` name inline) ---\n' +
59
+ blocks.join('') +
60
+ '\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/<path>`.'
61
+
62
+ const result = await runBenchRouterTurn(
63
+ {
64
+ routerBaseUrl,
65
+ routerKey,
66
+ profile: {
67
+ name: 'trata-hedge-solver',
68
+ harness: 'cli-base',
69
+ model: {
70
+ provider: 'tangle-router',
71
+ default: model,
72
+ metadata: { temperature, maxTokens },
73
+ },
74
+ prompt: { systemPrompt: instruction },
75
+ },
76
+ },
77
+ prompt,
78
+ )
79
+ writeFileSync(output, result.finalText)
80
+ process.stderr.write(`[solve] wrote ${result.finalText.length} chars -> ${output} (model=${model})\n`)
@@ -18,7 +18,7 @@ import type { Span } from '@tangle-network/agent-eval'
18
18
  import type { BenchTask } from './benchmarks/types'
19
19
  import { DEFAULT_BLENDER_DIRECTIVE } from './directives'
20
20
  import { runRefineLoop } from './refine-loop'
21
- import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
21
+ import { runBenchRouterTurn } from './router-turn'
22
22
 
23
23
  export { DEFAULT_BLENDER_DIRECTIVE } from './directives'
24
24
 
@@ -177,17 +177,27 @@ export async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig
177
177
  runShot: async (user, round, dir) => {
178
178
  const runnerPath = join(dir, 'runner.py')
179
179
  const scriptPath = join(dir, 'model.py')
180
- const { content, usage: u } = await routerChatWithUsage(
181
- cfg,
182
- [
183
- { role: 'system', content: directive },
184
- { role: 'user', content: user },
185
- ],
186
- { temperature: 0.3 },
180
+ const turn = await runBenchRouterTurn(
181
+ {
182
+ routerBaseUrl: cfg.routerBaseUrl,
183
+ routerKey: cfg.routerKey,
184
+ profile: {
185
+ name: 'blender-worker',
186
+ harness: 'cli-base',
187
+ model: {
188
+ provider: 'tangle-router',
189
+ default: cfg.model,
190
+ metadata: { temperature: 0.3 },
191
+ },
192
+ prompt: { systemPrompt: directive },
193
+ },
194
+ },
195
+ user,
187
196
  )
188
- if (u) {
189
- usage.input += u.input
190
- usage.output += u.output
197
+ const content = turn.finalText
198
+ if (turn.usage.tokensKnown !== false) {
199
+ usage.input += turn.usage.input
200
+ usage.output += turn.usage.output
191
201
  }
192
202
  const script = extractPy(content)
193
203
  trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
@@ -18,7 +18,7 @@
18
18
  import { readFile } from 'node:fs/promises'
19
19
  import type { Span } from '@tangle-network/agent-eval'
20
20
  import type { BenchTask } from './benchmarks/types'
21
- import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
21
+ import { runBenchRouterTurn } from './router-turn'
22
22
 
23
23
  export interface BrowserLocalConfig {
24
24
  routerBaseUrl: string
@@ -67,10 +67,23 @@ export async function solveBrowserLocal(task: BenchTask, cfg: BrowserLocalConfig
67
67
 
68
68
  trace.push({ spanId: 's-task', runId, kind: 'llm', name: 'web task', model: cfg.model, messages: [{ role: 'user', content: goal }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
69
69
 
70
- const { content, usage } = await routerChatWithUsage(cfg, [
71
- { role: 'system', content: directive },
72
- { role: 'user', content: task.prompt },
73
- ])
70
+ const turn = await runBenchRouterTurn(
71
+ {
72
+ routerBaseUrl: cfg.routerBaseUrl,
73
+ routerKey: cfg.routerKey,
74
+ profile: {
75
+ name: 'browser-local-worker',
76
+ model: { provider: 'tangle-router', default: cfg.model },
77
+ prompt: { systemPrompt: directive },
78
+ },
79
+ },
80
+ task.prompt,
81
+ )
82
+ const content = turn.finalText
83
+ const usage =
84
+ turn.usage.tokensKnown === false
85
+ ? undefined
86
+ : { input: turn.usage.input, output: turn.usage.output }
74
87
  const artifact = content.trim()
75
88
 
76
89
  const elementId = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact)?.[1] ?? ''
@@ -21,7 +21,7 @@ import type { Span } from '@tangle-network/agent-eval'
21
21
  import type { BenchTask } from './benchmarks/types'
22
22
  import { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
23
23
  import { runRefineLoop } from './refine-loop'
24
- import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
24
+ import { runBenchRouterTurn } from './router-turn'
25
25
 
26
26
  export { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
27
27
 
@@ -106,13 +106,22 @@ export async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig)
106
106
  runShot: async (user, round, dir) => {
107
107
  const scriptPath = join(dir, 'build.py')
108
108
  const stepPath = join(dir, 'output.step')
109
- const { content, usage: u } = await routerChatWithUsage(cfg, [
110
- { role: 'system', content: sys },
111
- { role: 'user', content: user },
112
- ])
113
- if (u) {
114
- usage.input += u.input
115
- usage.output += u.output
109
+ const turn = await runBenchRouterTurn(
110
+ {
111
+ routerBaseUrl: cfg.routerBaseUrl,
112
+ routerKey: cfg.routerKey,
113
+ profile: {
114
+ name: 'build123d-worker',
115
+ model: { provider: 'tangle-router', default: cfg.model },
116
+ prompt: { systemPrompt: sys },
117
+ },
118
+ },
119
+ user,
120
+ )
121
+ const content = turn.finalText
122
+ if (turn.usage.tokensKnown !== false) {
123
+ usage.input += turn.usage.input
124
+ usage.output += turn.usage.output
116
125
  }
117
126
  const source = extractPy(content)
118
127
  trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
package/src/worker-cad.ts CHANGED
@@ -17,12 +17,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
17
17
  import { tmpdir } from 'node:os'
18
18
  import { join } from 'node:path'
19
19
  import { promisify } from 'node:util'
20
- import { acquireSandbox, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
20
+ import { acquireSandbox } from '@tangle-network/agent-runtime/kernel'
21
21
  import { Sandbox } from '@tangle-network/sandbox'
22
22
  import type { Span } from '@tangle-network/agent-eval'
23
23
  import type { BenchTask } from './benchmarks/types'
24
24
  import { DEFAULT_CAD_DIRECTIVE, DEFAULT_CAD_SANDBOX_DIRECTIVE } from './directives'
25
25
  import { runRefineLoop } from './refine-loop'
26
+ import { runBenchRouterTurn } from './router-turn'
26
27
 
27
28
  export { DEFAULT_CAD_DIRECTIVE } from './directives'
28
29
 
@@ -138,13 +139,22 @@ export async function solveCadRefineLocal(task: BenchTask, cfg: CadLocalConfig):
138
139
  const scadPath = join(dir, 'model.scad')
139
140
  const stlPath = join(dir, 'model.stl')
140
141
  const pngPath = join(dir, 'model.png')
141
- const { content, usage: u } = await routerChatWithUsage(cfg, [
142
- { role: 'system', content: directive },
143
- { role: 'user', content: user },
144
- ])
145
- if (u) {
146
- usage.input += u.input
147
- usage.output += u.output
142
+ const turn = await runBenchRouterTurn(
143
+ {
144
+ routerBaseUrl: cfg.routerBaseUrl,
145
+ routerKey: cfg.routerKey,
146
+ profile: {
147
+ name: 'cad-local-worker',
148
+ model: { provider: 'tangle-router', default: cfg.model },
149
+ prompt: { systemPrompt: directive },
150
+ },
151
+ },
152
+ user,
153
+ )
154
+ const content = turn.finalText
155
+ if (turn.usage.tokensKnown !== false) {
156
+ usage.input += turn.usage.input
157
+ usage.output += turn.usage.output
148
158
  }
149
159
  const scad = extractScad(content)
150
160
  trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
@@ -253,10 +263,19 @@ export async function solveCadRefine(task: BenchTask, cfg: CadRefineConfig): Pro
253
263
  ? task.prompt
254
264
  : `Your previous OpenSCAD had this problem:\n${lastErr}\n\nHere is the previous source:\n${history[history.length - 1]?.artifact ?? ''}\n\nFix it so it compiles AND better matches the brief:\n${task.prompt}`,
255
265
  runShot: async (user, round, box) => {
256
- const { content: reply } = await routerChatWithUsage(cfg, [
257
- { role: 'system', content: sys },
258
- { role: 'user', content: user },
259
- ])
266
+ const turn = await runBenchRouterTurn(
267
+ {
268
+ routerBaseUrl: cfg.routerBaseUrl,
269
+ routerKey: cfg.routerKey,
270
+ profile: {
271
+ name: 'cad-sandbox-worker',
272
+ model: { provider: 'tangle-router', default: cfg.model },
273
+ prompt: { systemPrompt: sys },
274
+ },
275
+ },
276
+ user,
277
+ )
278
+ const reply = turn.finalText
260
279
  const scad = extractScad(reply)
261
280
  trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: reply.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
262
281
  trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.scad', content: scad }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
package/src/worker.ts CHANGED
@@ -89,7 +89,7 @@ export async function solveShot(
89
89
 
90
90
  // Cold-start-resilient via the shared lineage layer (a gateway-timed-out create is
91
91
  // recovered by name lookup). The inline profile + backend override is the same
92
- // generic AgentRunSpec the runLoop kernel boots against the real sandbox.
92
+ // generic AgentRunSpec the runAgentRounds kernel boots against the real sandbox.
93
93
  const controller = new AbortController()
94
94
  const timer = cfg.timeoutMs ? setTimeout(() => controller.abort(), cfg.timeoutMs) : undefined
95
95
  const agentRun: AgentRunSpec<string> = {
@@ -66,6 +66,15 @@ _USAGE_OUT_PATH = f"{_CONTAINER_DIR}/opencode-events.jsonl"
66
66
  _USAGE_ERR_PATH = f"{_CONTAINER_DIR}/opencode-run.err"
67
67
 
68
68
 
69
+ REFINE_DIRECTIVE = (
70
+ "REFINE PASS — your previous attempt failed the task's automated tests. "
71
+ "Use the evidence below to correct the concrete failure without discarding work "
72
+ "that was already right. Verify the task's success criteria before finishing.\n\n"
73
+ "<previous_attempt>\n{prior}\n</previous_attempt>\n\n"
74
+ "--- ORIGINAL TASK ---\n{instruction}"
75
+ )
76
+
77
+
69
78
  class OpenCodeRouterAgent(OpenCodeAgent):
70
79
  """`OpenCodeAgent` + Tangle-router routing + real token metering.
71
80
 
@@ -77,27 +86,71 @@ class OpenCodeRouterAgent(OpenCodeAgent):
77
86
  a real `AgentResult`.
78
87
  """
79
88
 
80
- def __init__(self, model_name: str, *args, **kwargs):
81
- # The stock __init__ does `self._provider, _ = model_name.split("/")`, which
82
- # raises on a 3-segment or bare id. Derive the provider robustly ourselves,
83
- # then hand the base class a value it can split without raising.
84
- self._model_name = model_name
85
- provider, _, sub = model_name.partition("/")
89
+ def __init__(
90
+ self,
91
+ model_name: str,
92
+ profile_path: str,
93
+ prior_attempt_hex: str | None = None,
94
+ *args,
95
+ **kwargs,
96
+ ):
97
+ self._profile = json.loads(Path(profile_path).read_text())
98
+ if self._profile.get("harness") != "opencode":
99
+ raise ValueError("AgentProfile.harness must be 'opencode'")
100
+ model = self._profile.get("model")
101
+ if not isinstance(model, dict):
102
+ raise ValueError("AgentProfile.model is required")
103
+ provider = model.get("provider")
104
+ declared = model.get("default")
105
+ if not isinstance(provider, str) or not provider.strip():
106
+ raise ValueError("AgentProfile.model.provider must be explicit")
107
+ if not isinstance(declared, str) or not declared.strip() or declared == "runtime-selected":
108
+ raise ValueError("AgentProfile.model.default must be concrete")
109
+ wire_model = declared if "/" in declared else f"{provider}/{declared}"
110
+ if model_name != wire_model:
111
+ raise ValueError(
112
+ f"Terminal-Bench model {model_name!r} conflicts with AgentProfile {wire_model!r}"
113
+ )
114
+ self._model_name = wire_model
86
115
  self._provider = provider
87
- # Model id opencode sees, minus the provider segment (e.g. "glm-5.2").
88
- self._provider_model = sub or provider
116
+ self._provider_model = declared.split("/", 1)[-1]
117
+ self._prior_attempt = self._decode_prior(prior_attempt_hex)
118
+ prompt = self._profile.get("prompt")
119
+ self._profile_instructions: list[str] = []
120
+ if isinstance(prompt, dict):
121
+ system = prompt.get("systemPrompt")
122
+ if isinstance(system, str) and system.strip():
123
+ self._profile_instructions.append(system.strip())
124
+ instructions = prompt.get("instructions")
125
+ if isinstance(instructions, list):
126
+ self._profile_instructions.extend(
127
+ value.strip()
128
+ for value in instructions
129
+ if isinstance(value, str) and value.strip()
130
+ )
89
131
  self._router_base_url = (
90
132
  os.environ.get("OPENAI_BASE_URL")
91
133
  or os.environ.get("ROUTER_BASE_URL")
92
134
  or DEFAULT_ROUTER_BASE_URL
93
135
  )
94
136
  self._router_api_key = os.environ.get("OPENAI_API_KEY", "")
95
- # Call the grandparent (AbstractInstalledAgent) init via OpenCodeAgent, but
96
- # avoid the base split-crash by not delegating provider derivation to it.
97
- super(OpenCodeAgent, self).__init__(*args, **kwargs)
98
- self._version = kwargs.get("version", "latest")
137
+ super().__init__(wire_model, *args, **kwargs)
99
138
  self._logger = logger.getChild(__name__)
100
139
 
140
+ @staticmethod
141
+ def _decode_prior(value: str | None) -> str:
142
+ if not value:
143
+ return ""
144
+ raw = str(value).strip()
145
+ if raw.startswith("h"):
146
+ raw = raw[1:]
147
+ if not raw:
148
+ return ""
149
+ try:
150
+ return bytes.fromhex(raw).decode("utf-8").strip()
151
+ except ValueError as error:
152
+ raise ValueError("prior_attempt_hex must be 'h' + hex-encoded UTF-8") from error
153
+
101
154
  @staticmethod
102
155
  def name() -> str:
103
156
  return "agent-runtime-opencode-router"
@@ -136,16 +189,6 @@ class OpenCodeRouterAgent(OpenCodeAgent):
136
189
  "OPENAI_BASE_URL": self._router_base_url,
137
190
  "OPENCODE_CONFIG": _CONFIG_PATH,
138
191
  }
139
- # Pass through any provider-native key the user already exported, so the same
140
- # agent also works when pointed at a vendor endpoint rather than the router.
141
- for passthrough in (
142
- "ANTHROPIC_API_KEY",
143
- "DEEPSEEK_API_KEY",
144
- "GROQ_API_KEY",
145
- "ZAI_API_KEY",
146
- ):
147
- if passthrough in os.environ:
148
- env.setdefault(passthrough, os.environ[passthrough])
149
192
  return {k: v for k, v in env.items() if v}
150
193
 
151
194
  def _build_router_config(self) -> str:
@@ -159,14 +202,7 @@ class OpenCodeRouterAgent(OpenCodeAgent):
159
202
  "$schema": "https://opencode.ai/config.json",
160
203
  # Headless benchmark runs cannot answer interactive permission prompts.
161
204
  # Keep this identical for raw and supervisor arms.
162
- "permission": {
163
- "edit": "allow",
164
- "bash": "allow",
165
- "webfetch": "allow",
166
- "read": "allow",
167
- "write": "allow",
168
- "external_directory": "allow",
169
- },
205
+ "permission": self._profile.get("permission", {}),
170
206
  "provider": {
171
207
  self._provider: {
172
208
  "npm": "@ai-sdk/openai-compatible",
@@ -184,25 +220,24 @@ class OpenCodeRouterAgent(OpenCodeAgent):
184
220
  return json.dumps(config)
185
221
 
186
222
  def _run_agent_commands(self, instruction: str):
187
- """Same command as the stock agent, but `--format json` with the event
188
- stream captured to a file so we can meter token usage after the run."""
223
+ """Instrument Terminal-Bench's stock command without selecting a model here."""
189
224
  from terminal_bench.terminal.models import TerminalCommand
190
225
 
191
- escaped_instruction = shlex.quote(instruction)
192
- command = (
193
- f"opencode --model {self._model_name} --format json "
194
- f"run {escaped_instruction} "
195
- f"> {_USAGE_OUT_PATH} 2> {_USAGE_ERR_PATH}"
196
- )
197
- return [
198
- TerminalCommand(
226
+ instrumented = []
227
+ for base in super()._run_agent_commands(instruction):
228
+ marker = " run "
229
+ if marker not in base.command:
230
+ raise ValueError("stock OpenCodeAgent command has no run subcommand")
231
+ command = base.command.replace(marker, " --format json run ", 1)
232
+ command += f" > {_USAGE_OUT_PATH} 2> {_USAGE_ERR_PATH}"
233
+ instrumented.append(TerminalCommand(
199
234
  command=command,
200
235
  min_timeout_sec=0.0,
201
236
  max_timeout_sec=float("inf"),
202
237
  block=True,
203
238
  append_enter=True,
204
- )
205
- ]
239
+ ))
240
+ return instrumented
206
241
 
207
242
  def _write_config_to_container(self, session: TmuxSession) -> None:
208
243
  config_json = self._build_router_config()
@@ -365,6 +400,13 @@ class OpenCodeRouterAgent(OpenCodeAgent):
365
400
  "OPENAI_API_KEY (router key) is not set; opencode has no credential "
366
401
  "to reach the router."
367
402
  )
403
+ if self._profile_instructions:
404
+ instruction = "\n\n".join([*self._profile_instructions, instruction])
405
+ if self._prior_attempt:
406
+ instruction = REFINE_DIRECTIVE.format(
407
+ prior=self._prior_attempt,
408
+ instruction=instruction,
409
+ )
368
410
  # Publish container id + inject router config BEFORE the base flow runs the
369
411
  # agent. The base perform_task copies the install script into _CONTAINER_DIR,
370
412
  # sources setup-env.sh (which exports OPENCODE_CONFIG), installs opencode, and
@@ -1 +0,0 @@
1
- {"version":3,"file":"cadbench-BrpwOU6A.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n try {\n const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n return { code: 0, stdout, stderr }\n } catch (err) {\n const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n for c in o.bound_box:\n w=o.matrix_world @ mathutils.Vector(c)\n for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n az=math.radians(40 + v*360.0/nviews)\n cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n * Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n const views = Math.max(1, opts.views ?? 4)\n const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n try {\n await writeFile(runnerPath, RUNNER_PY)\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n const built = /RENDER_DONE/.test(out)\n if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n const renders: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n return { built: renders.length > 0, renders }\n } finally {\n await rm(dir, { recursive: true, force: true }).catch(() => {})\n }\n}\n\nexport interface BlenderLocalConfig {\n routerBaseUrl: string\n routerKey: string\n model: string\n rounds?: number\n /** N standardized views to render (CADBench uses 4). Default 4. */\n views?: number\n /** The bpy authoring directive — the GEPA-optimizable surface. */\n directive?: string\n}\n\nexport interface BlenderShot {\n /** The bpy script the agent wrote — the artifact. */\n artifact: string\n /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n renders: string[]\n trace: Span[]\n usage: { input: number; output: number }\n ok: boolean\n /** True if the script executed and produced at least one mesh. */\n built: boolean\n detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n const rounds = Math.max(1, cfg.rounds ?? 2)\n const views = Math.max(1, cfg.views ?? 4)\n const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n const trace: Span[] = []\n const runId = `cadbench-${task.id}`\n let ts = Date.now()\n const tick = () => (ts += 1)\n const usage = { input: 0, output: 0 }\n // Carried across rounds in closures (the round Artifact is the bpy script; the\n // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n let renders: string[] = []\n let built = false\n let lastErr = ''\n\n trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n // lastErr + the prior script verbatim.\n const res = await runRefineLoop<string, string>({\n rounds,\n setup: async () => {\n const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n return dir\n },\n prompt: (round, history) =>\n round === 1\n ? task.prompt\n : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n runShot: async (user, round, dir) => {\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n const { content, usage: u } = await routerChatWithUsage(\n cfg,\n [\n { role: 'system', content: directive },\n { role: 'user', content: user },\n ],\n { temperature: 0.3 },\n )\n if (u) {\n usage.input += u.input\n usage.output += u.output\n }\n const script = extractPy(content)\n trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n built = /RENDER_DONE/.test(out)\n lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n if (built) {\n const collected: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n renders = collected\n // first view carries the screen span (run-capsule reveal)\n trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n built = collected.length > 0\n }\n return { artifact: script }\n },\n judge: async () => ({ valid: built }),\n teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n })\n\n const script = res.final.artifact\n return {\n artifact: script,\n renders,\n trace,\n usage,\n ok: script.trim().length > 0,\n built,\n detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n name: string\n type: string\n criteria: string[]\n}\n\nfunction must(name: string): string {\n const v = process.env[name]\n if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n * criteria → a JSON array of booleans (true = satisfied). Faithful to the\n * paper's combined image+script evaluation. Throws on transport failure (never\n * a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n instruction: string,\n script: string,\n criteria: string[],\n renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n const key = must('TANGLE_API_KEY')\n const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n const text =\n `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n const content: unknown[] = [{ type: 'text', text }]\n for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n const res = await fetch(`${base}/chat/completions`, {\n method: 'POST',\n headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n })\n if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n const raw = data.choices?.[0]?.message?.content ?? ''\n const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n let arr: unknown\n try {\n arr = JSON.parse(m[0].toLowerCase())\n } catch {\n return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n }\n const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n // Pad/truncate to criteria length (a short array scores the missing as fail).\n const passed = criteria.map((_, i) => bools[i] === true)\n return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n async function load(): Promise<typeof cache & object> {\n if (cache) return cache\n const path = process.env.CADBENCH_PATH\n if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n const text = await readFile(path, 'utf8')\n cache = text\n .split('\\n')\n .filter((l) => l.trim())\n .map((l) => {\n const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n })\n return cache\n }\n\n return {\n name: 'cadbench',\n\n async preflight() {\n const { execFile } = await import('node:child_process')\n const { promisify } = await import('node:util')\n const exec = promisify(execFile)\n try {\n await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n } catch (err) {\n throw new Error(\n `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n )\n }\n await load()\n },\n\n async loadTasks(opts: LoadOptions = {}) {\n let rows = await load()\n if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n // TYPE filter (Simulative|Wild) via env, applied before limit.\n const t = process.env.CADBENCH_TYPE\n if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n if (opts.limit != null) rows = rows.slice(0, opts.limit)\n return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n },\n\n async goldArtifact() {\n return undefined // no reference bpy script ships with the benchmark\n },\n\n async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n const meta = task.metadata as unknown as CadBenchMeta\n const criteria = meta.criteria ?? []\n if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n const r = await renderBpy(artifact, { views: 4 })\n if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n const score = passed.filter(Boolean).length / criteria.length\n return { resolved: score === 1, score, detail: note }\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;AC1FA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CACjF,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,oBAAoB;EAClD,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU;EAAM;EAC9E,MAAM,KAAK,UAAU;GAAE;GAAO,YAAY;GAAM,aAAa;GAAG,UAAU,CAAC;IAAE,MAAM;IAAQ;GAAQ,CAAC;EAAE,CAAC;CACzG,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAEhG,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,UAAU,EAAE,EAAE,SAAS,WAAW;CACnD,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}