@tangle-network/agent-bench 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/HARNESS.md +1 -1
- package/dist/adapters.js +2 -2
- package/dist/benchmarks/appworld.d.ts +61 -2
- package/dist/benchmarks/appworld.js +111 -40
- package/dist/benchmarks/appworld.js.map +1 -1
- package/dist/benchmarks/cadbench.js +1 -1
- package/dist/benchmarks/cadgenbench.js +1 -1
- package/dist/benchmarks/finresearchbench.js +16 -22
- package/dist/benchmarks/finresearchbench.js.map +1 -1
- package/dist/benchmarks/finsearchcomp.js +16 -22
- package/dist/benchmarks/finsearchcomp.js.map +1 -1
- package/dist/benchmarks/frames.js +18 -20
- package/dist/benchmarks/frames.js.map +1 -1
- package/dist/benchmarks/simpleqa.js +15 -19
- package/dist/benchmarks/simpleqa.js.map +1 -1
- package/dist/benchmarks/trata-hedge.js +20 -23
- package/dist/benchmarks/trata-hedge.js.map +1 -1
- package/dist/{cadbench-BrpwOU6A.js → cadbench-BLSyxR1N.js} +21 -20
- package/dist/cadbench-BLSyxR1N.js.map +1 -0
- package/dist/{cadgenbench-DF7hYHdl.js → cadgenbench-x2OFkf8y.js} +2 -2
- package/dist/cadgenbench-x2OFkf8y.js.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.js +10 -9
- package/dist/index.js.map +1 -1
- package/dist/router-turn-C2wMiDoo.js +31 -0
- package/dist/router-turn-C2wMiDoo.js.map +1 -0
- package/package.json +7 -7
- package/scripts/appworld_driver.py +1 -170
- package/scripts/run-package-tests.mjs +56 -36
- package/scripts/run-package-tests.test.mjs +33 -0
- package/scripts/trata-hedge/README.md +3 -3
- package/scripts/trata-hedge/run.sh +6 -2
- package/src/aec-gate.mts +30 -9
- package/src/agent-graphs-gen2.mts +523 -0
- package/src/agent-graphs-gen3.mts +660 -0
- package/src/agent-graphs-improve/offline-seams.mts +128 -0
- package/src/agent-graphs-improve.mts +747 -0
- package/src/atom-humaneval.mts +76 -55
- package/src/atom-mcp-e2e.mts +27 -13
- package/src/benchmarks/appworld.test.mts +119 -1
- package/src/benchmarks/appworld.ts +156 -33
- package/src/benchmarks/cadbench.ts +21 -8
- package/src/benchmarks/finresearchbench.ts +20 -16
- package/src/benchmarks/finsearchcomp.ts +20 -20
- package/src/benchmarks/frames.ts +19 -18
- package/src/benchmarks/simpleqa.ts +19 -17
- package/src/benchmarks/trata-hedge.ts +26 -18
- package/src/browser/ui-reviewer.ts +1 -1
- package/src/clbench-context-gate.mts +45 -9
- package/src/cloud-loop.mts +24 -6
- package/src/commit0-env-run.mts +6 -3
- package/src/commit0-env.ts +2 -3
- package/src/commit0-gate.mts +53 -24
- package/src/corpus.ts +3 -3
- package/src/david-attribution.mts +28 -9
- package/src/david-goliath.mts +31 -12
- package/src/egress-probe.mts +3 -3
- package/src/examples/lean-proof-gate.mts +10 -3
- package/src/examples/math-demo.mts +10 -3
- package/src/examples/strategy-demo.mts +10 -4
- package/src/fleet.mts +30 -8
- package/src/gate-cli.mts +9 -4
- package/src/gate.test.mts +7 -2
- package/src/gate.ts +23 -75
- package/src/generate-eval/certify.ts +13 -5
- package/src/hev-eval.mts +39 -10
- package/src/hev-improve.mts +28 -16
- package/src/hev-structural.mts +28 -20
- package/src/humaneval-gate.mts +24 -8
- package/src/humaneval-object-ablation.mts +52 -14
- package/src/humaneval-repair-gate.mts +53 -14
- package/src/mbpp-structural.mts +28 -20
- package/src/mcp-mount-probe.mts +23 -11
- package/src/official-optimizer-config.mts +61 -12
- package/src/official-optimizer-config.test.mts +72 -3
- package/src/quant-arena/profiles/quant-researcher.profile.json +6 -1
- package/src/quant-arena/quant-loop.mts +88 -61
- package/src/research-shot.ts +21 -11
- package/src/resolve-client.ts +6 -8
- package/src/rollout-ledger/backfill-swe-arena.test.mts +7 -2
- package/src/router-executor.ts +1 -1
- package/src/router-turn.ts +135 -0
- package/src/run-benchmarks.ts +7 -2
- package/src/sandbox-run.ts +23 -41
- package/src/search-bench/bridge.ts +66 -8
- package/src/search-bench/parametric-check.mts +19 -4
- package/src/search-bench/profiles.ts +24 -9
- package/src/search-bench/run.mts +3 -4
- package/src/smoke-structural-rollout.mts +21 -5
- package/src/supervisor-arena.mts +28 -20
- package/src/swe-arena/arms.ts +73 -15
- package/src/swe-arena/capacity.ts +44 -41
- package/src/swe-arena/diagnosis-ensemble.ts +51 -48
- package/src/swe-arena/execution.test.mts +12 -12
- package/src/swe-arena/gepa-seat.mts +12 -5
- package/src/swe-arena/gepa-seat.test.mts +45 -33
- package/src/swe-arena/outer-loop.mts +94 -70
- package/src/swe-arena/premeasured-from-cells.mts +16 -1
- package/src/swe-arena/premeasured-from-cells.test.mts +21 -1
- package/src/swe-arena/profiles/deepseek-author.profile.json +12 -0
- package/src/swe-arena/profiles/default-author.profile.json +9 -1
- package/src/swe-arena/proposer-fanout.mts +52 -99
- package/src/swe-arena/proposer-fanout.test.mts +92 -57
- package/src/swe-arena/proposer-provenance.mts +1 -1
- package/src/swe-code-improve.mts +30 -52
- package/src/swe-emit-patch.mts +26 -5
- package/src/swe-improve.mts +27 -8
- package/src/swe-jail.test.ts +128 -0
- package/src/swe-jail.ts +106 -22
- package/src/swe-local-proof.mts +27 -5
- package/src/swe-repro-calibrate.mts +5 -0
- package/src/swe-self-improve.mts +45 -10
- package/src/swe-stream.mts +68 -6
- package/src/swe-structural.mts +46 -7
- package/src/tb-container-executor.mts +38 -6
- package/src/tb-container-executor.test.mts +11 -3
- package/src/tb-supervisor-sidecar.mts +11 -27
- package/src/terminal-compare.ts +38 -18
- package/src/trata-gate.mts +26 -26
- package/src/trata-gepa.mts +25 -16
- package/src/trata-hedge-solve.mts +80 -0
- package/src/worker-blender.ts +21 -11
- package/src/worker-browser.ts +18 -5
- package/src/worker-build123d.ts +17 -8
- package/src/worker-cad.ts +31 -12
- package/src/worker.ts +1 -1
- package/tb_agents/opencode_router_agent.py +84 -42
- package/dist/cadbench-BrpwOU6A.js.map +0 -1
- package/dist/cadgenbench-DF7hYHdl.js.map +0 -1
- package/scripts/trata-hedge/solve.py +0 -83
package/src/trata-gate.mts
CHANGED
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
import { appendFileSync } from 'node:fs'
|
|
31
31
|
import { resolveAdapter } from './adapters'
|
|
32
32
|
import type { BenchScore, BenchTask } from './benchmarks/types'
|
|
33
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
33
34
|
import { runPool } from './run-pool'
|
|
34
35
|
|
|
35
36
|
function must(name: string): string {
|
|
@@ -61,33 +62,32 @@ async function workerComplete(
|
|
|
61
62
|
cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
|
|
62
63
|
): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> {
|
|
63
64
|
const startedAt = Date.now()
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
const answer = j.choices?.[0]?.message?.content ?? ''
|
|
65
|
+
const result = await runBenchRouterTurn(
|
|
66
|
+
{
|
|
67
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
68
|
+
routerKey: cfg.routerKey,
|
|
69
|
+
profile: {
|
|
70
|
+
name: 'trata-financial-analyst',
|
|
71
|
+
harness: 'cli-base',
|
|
72
|
+
model: {
|
|
73
|
+
provider: 'tangle-router',
|
|
74
|
+
default: cfg.model,
|
|
75
|
+
metadata: {
|
|
76
|
+
temperature: 0,
|
|
77
|
+
maxTokens: Number(process.env.WORKER_MAX_TOKENS ?? 4096),
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
prompt: { systemPrompt: ANALYST_SYSTEM },
|
|
81
|
+
},
|
|
82
|
+
timeoutMs: cfg.timeoutMs,
|
|
83
|
+
},
|
|
84
|
+
task.prompt,
|
|
85
|
+
)
|
|
86
|
+
if (result.usage.tokensKnown === false) throw new Error('worker provider omitted token usage')
|
|
87
87
|
return {
|
|
88
|
-
answer,
|
|
89
|
-
inputTokens:
|
|
90
|
-
outputTokens:
|
|
88
|
+
answer: result.finalText,
|
|
89
|
+
inputTokens: result.usage.input,
|
|
90
|
+
outputTokens: result.usage.output,
|
|
91
91
|
durationMs: Date.now() - startedAt,
|
|
92
92
|
}
|
|
93
93
|
}
|
package/src/trata-gepa.mts
CHANGED
|
@@ -60,6 +60,7 @@ import {
|
|
|
60
60
|
officialOptimizerModel,
|
|
61
61
|
requiredTokenPricing,
|
|
62
62
|
} from './official-optimizer-config.mjs'
|
|
63
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
63
64
|
|
|
64
65
|
interface TrataScenario extends Scenario {
|
|
65
66
|
task: BenchTask
|
|
@@ -122,23 +123,31 @@ async function chatComplete(
|
|
|
122
123
|
messages: Array<{ role: string; content: string }>,
|
|
123
124
|
signal: AbortSignal,
|
|
124
125
|
): Promise<{ content: string; usage?: { input: number; output: number } }> {
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
126
|
+
const system = messages.find((message) => message.role === 'system')?.content
|
|
127
|
+
const result = await runBenchRouterTurn(
|
|
128
|
+
{
|
|
129
|
+
routerBaseUrl: baseUrl,
|
|
130
|
+
routerKey: key,
|
|
131
|
+
profile: {
|
|
132
|
+
name: 'trata-gepa-worker',
|
|
133
|
+
harness: 'cli-base',
|
|
134
|
+
model: {
|
|
135
|
+
provider: 'tangle-router',
|
|
136
|
+
default: model,
|
|
137
|
+
metadata: { temperature: 0, maxTokens },
|
|
138
|
+
},
|
|
139
|
+
...(system ? { prompt: { systemPrompt: system } } : {}),
|
|
140
|
+
},
|
|
141
|
+
signal,
|
|
142
|
+
},
|
|
143
|
+
{ messages: messages.filter((message) => message.role !== 'system') },
|
|
144
|
+
)
|
|
145
|
+
return {
|
|
146
|
+
content: result.finalText,
|
|
147
|
+
...(result.usage.tokensKnown === false
|
|
148
|
+
? {}
|
|
149
|
+
: { usage: { input: result.usage.input, output: result.usage.output } }),
|
|
135
150
|
}
|
|
136
|
-
const content = j.choices?.[0]?.message?.content ?? ''
|
|
137
|
-
const usage =
|
|
138
|
-
j.usage?.prompt_tokens != null
|
|
139
|
-
? { input: j.usage.prompt_tokens, output: j.usage.completion_tokens ?? 0 }
|
|
140
|
-
: undefined
|
|
141
|
-
return { content, usage }
|
|
142
151
|
}
|
|
143
152
|
|
|
144
153
|
async function main(): Promise<void> {
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** Single-shot lower-bound solver for trata-hedge-bench. */
|
|
2
|
+
import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
|
|
3
|
+
import { join, relative, resolve } from 'node:path'
|
|
4
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
5
|
+
|
|
6
|
+
const environment = process.argv[2]
|
|
7
|
+
if (!environment) throw new Error('usage: solve.mts <env-dir> [out.txt]')
|
|
8
|
+
const output = process.argv[3] ?? '/tmp/thb-answer.txt'
|
|
9
|
+
const model = process.env.WORKER_MODEL
|
|
10
|
+
if (!model) throw new Error('WORKER_MODEL is required')
|
|
11
|
+
const dataBudget = Number(process.env.DATA_BUDGET ?? 160_000)
|
|
12
|
+
const maxTokens = Number(process.env.MAX_TOKENS ?? 6_000)
|
|
13
|
+
const temperature = Number(process.env.TEMPERATURE ?? 0.5)
|
|
14
|
+
const routerKey = process.env.TANGLE_API_KEY
|
|
15
|
+
if (!routerKey) throw new Error('TANGLE_API_KEY is required')
|
|
16
|
+
const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
|
|
17
|
+
|
|
18
|
+
for (const [name, value] of [
|
|
19
|
+
['DATA_BUDGET', dataBudget],
|
|
20
|
+
['MAX_TOKENS', maxTokens],
|
|
21
|
+
] as const) {
|
|
22
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`)
|
|
23
|
+
}
|
|
24
|
+
if (!Number.isFinite(temperature)) throw new Error('TEMPERATURE must be finite')
|
|
25
|
+
|
|
26
|
+
const root = resolve(environment)
|
|
27
|
+
const dataDir = join(root, 'environment', 'data')
|
|
28
|
+
const instruction = readFileSync(join(root, 'instruction.md'), 'utf8')
|
|
29
|
+
|
|
30
|
+
function filesUnder(dir: string): string[] {
|
|
31
|
+
return readdirSync(dir)
|
|
32
|
+
.flatMap((name) => {
|
|
33
|
+
const path = join(dir, name)
|
|
34
|
+
return statSync(path).isDirectory() ? filesUnder(path) : [path]
|
|
35
|
+
})
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function rank(path: string): number {
|
|
39
|
+
if (path.includes('earnings_call')) return 0
|
|
40
|
+
if (path.includes('financials')) return 1
|
|
41
|
+
if (path.includes('company_profiles')) return 2
|
|
42
|
+
if (path.includes('press_releases')) return 3
|
|
43
|
+
return 4
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let used = 0
|
|
47
|
+
const blocks: string[] = []
|
|
48
|
+
const files = filesUnder(dataDir).sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
|
|
49
|
+
for (const path of files) {
|
|
50
|
+
const content = readFileSync(path, 'utf8')
|
|
51
|
+
if (used + content.length > dataBudget) continue
|
|
52
|
+
used += content.length
|
|
53
|
+
blocks.push(`\n=== FILE: data/${relative(dataDir, path)} ===\n${content}`)
|
|
54
|
+
}
|
|
55
|
+
process.stderr.write(`[solve] ${blocks.length}/${files.length} files in context (${used} chars)\n`)
|
|
56
|
+
|
|
57
|
+
const prompt =
|
|
58
|
+
'--- AVAILABLE DATA (cite files by their `data/<path>` name inline) ---\n' +
|
|
59
|
+
blocks.join('') +
|
|
60
|
+
'\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/<path>`.'
|
|
61
|
+
|
|
62
|
+
const result = await runBenchRouterTurn(
|
|
63
|
+
{
|
|
64
|
+
routerBaseUrl,
|
|
65
|
+
routerKey,
|
|
66
|
+
profile: {
|
|
67
|
+
name: 'trata-hedge-solver',
|
|
68
|
+
harness: 'cli-base',
|
|
69
|
+
model: {
|
|
70
|
+
provider: 'tangle-router',
|
|
71
|
+
default: model,
|
|
72
|
+
metadata: { temperature, maxTokens },
|
|
73
|
+
},
|
|
74
|
+
prompt: { systemPrompt: instruction },
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
prompt,
|
|
78
|
+
)
|
|
79
|
+
writeFileSync(output, result.finalText)
|
|
80
|
+
process.stderr.write(`[solve] wrote ${result.finalText.length} chars -> ${output} (model=${model})\n`)
|
package/src/worker-blender.ts
CHANGED
|
@@ -18,7 +18,7 @@ import type { Span } from '@tangle-network/agent-eval'
|
|
|
18
18
|
import type { BenchTask } from './benchmarks/types'
|
|
19
19
|
import { DEFAULT_BLENDER_DIRECTIVE } from './directives'
|
|
20
20
|
import { runRefineLoop } from './refine-loop'
|
|
21
|
-
import {
|
|
21
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
22
22
|
|
|
23
23
|
export { DEFAULT_BLENDER_DIRECTIVE } from './directives'
|
|
24
24
|
|
|
@@ -177,17 +177,27 @@ export async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig
|
|
|
177
177
|
runShot: async (user, round, dir) => {
|
|
178
178
|
const runnerPath = join(dir, 'runner.py')
|
|
179
179
|
const scriptPath = join(dir, 'model.py')
|
|
180
|
-
const
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
180
|
+
const turn = await runBenchRouterTurn(
|
|
181
|
+
{
|
|
182
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
183
|
+
routerKey: cfg.routerKey,
|
|
184
|
+
profile: {
|
|
185
|
+
name: 'blender-worker',
|
|
186
|
+
harness: 'cli-base',
|
|
187
|
+
model: {
|
|
188
|
+
provider: 'tangle-router',
|
|
189
|
+
default: cfg.model,
|
|
190
|
+
metadata: { temperature: 0.3 },
|
|
191
|
+
},
|
|
192
|
+
prompt: { systemPrompt: directive },
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
user,
|
|
187
196
|
)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
usage.
|
|
197
|
+
const content = turn.finalText
|
|
198
|
+
if (turn.usage.tokensKnown !== false) {
|
|
199
|
+
usage.input += turn.usage.input
|
|
200
|
+
usage.output += turn.usage.output
|
|
191
201
|
}
|
|
192
202
|
const script = extractPy(content)
|
|
193
203
|
trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
package/src/worker-browser.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import { readFile } from 'node:fs/promises'
|
|
19
19
|
import type { Span } from '@tangle-network/agent-eval'
|
|
20
20
|
import type { BenchTask } from './benchmarks/types'
|
|
21
|
-
import {
|
|
21
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
22
22
|
|
|
23
23
|
export interface BrowserLocalConfig {
|
|
24
24
|
routerBaseUrl: string
|
|
@@ -67,10 +67,23 @@ export async function solveBrowserLocal(task: BenchTask, cfg: BrowserLocalConfig
|
|
|
67
67
|
|
|
68
68
|
trace.push({ spanId: 's-task', runId, kind: 'llm', name: 'web task', model: cfg.model, messages: [{ role: 'user', content: goal }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
|
69
69
|
|
|
70
|
-
const
|
|
71
|
-
{
|
|
72
|
-
|
|
73
|
-
|
|
70
|
+
const turn = await runBenchRouterTurn(
|
|
71
|
+
{
|
|
72
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
73
|
+
routerKey: cfg.routerKey,
|
|
74
|
+
profile: {
|
|
75
|
+
name: 'browser-local-worker',
|
|
76
|
+
model: { provider: 'tangle-router', default: cfg.model },
|
|
77
|
+
prompt: { systemPrompt: directive },
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
task.prompt,
|
|
81
|
+
)
|
|
82
|
+
const content = turn.finalText
|
|
83
|
+
const usage =
|
|
84
|
+
turn.usage.tokensKnown === false
|
|
85
|
+
? undefined
|
|
86
|
+
: { input: turn.usage.input, output: turn.usage.output }
|
|
74
87
|
const artifact = content.trim()
|
|
75
88
|
|
|
76
89
|
const elementId = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact)?.[1] ?? ''
|
package/src/worker-build123d.ts
CHANGED
|
@@ -21,7 +21,7 @@ import type { Span } from '@tangle-network/agent-eval'
|
|
|
21
21
|
import type { BenchTask } from './benchmarks/types'
|
|
22
22
|
import { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
|
|
23
23
|
import { runRefineLoop } from './refine-loop'
|
|
24
|
-
import {
|
|
24
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
25
25
|
|
|
26
26
|
export { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
|
|
27
27
|
|
|
@@ -106,13 +106,22 @@ export async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig)
|
|
|
106
106
|
runShot: async (user, round, dir) => {
|
|
107
107
|
const scriptPath = join(dir, 'build.py')
|
|
108
108
|
const stepPath = join(dir, 'output.step')
|
|
109
|
-
const
|
|
110
|
-
{
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
109
|
+
const turn = await runBenchRouterTurn(
|
|
110
|
+
{
|
|
111
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
112
|
+
routerKey: cfg.routerKey,
|
|
113
|
+
profile: {
|
|
114
|
+
name: 'build123d-worker',
|
|
115
|
+
model: { provider: 'tangle-router', default: cfg.model },
|
|
116
|
+
prompt: { systemPrompt: sys },
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
user,
|
|
120
|
+
)
|
|
121
|
+
const content = turn.finalText
|
|
122
|
+
if (turn.usage.tokensKnown !== false) {
|
|
123
|
+
usage.input += turn.usage.input
|
|
124
|
+
usage.output += turn.usage.output
|
|
116
125
|
}
|
|
117
126
|
const source = extractPy(content)
|
|
118
127
|
trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
package/src/worker-cad.ts
CHANGED
|
@@ -17,12 +17,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
|
17
17
|
import { tmpdir } from 'node:os'
|
|
18
18
|
import { join } from 'node:path'
|
|
19
19
|
import { promisify } from 'node:util'
|
|
20
|
-
import { acquireSandbox
|
|
20
|
+
import { acquireSandbox } from '@tangle-network/agent-runtime/kernel'
|
|
21
21
|
import { Sandbox } from '@tangle-network/sandbox'
|
|
22
22
|
import type { Span } from '@tangle-network/agent-eval'
|
|
23
23
|
import type { BenchTask } from './benchmarks/types'
|
|
24
24
|
import { DEFAULT_CAD_DIRECTIVE, DEFAULT_CAD_SANDBOX_DIRECTIVE } from './directives'
|
|
25
25
|
import { runRefineLoop } from './refine-loop'
|
|
26
|
+
import { runBenchRouterTurn } from './router-turn'
|
|
26
27
|
|
|
27
28
|
export { DEFAULT_CAD_DIRECTIVE } from './directives'
|
|
28
29
|
|
|
@@ -138,13 +139,22 @@ export async function solveCadRefineLocal(task: BenchTask, cfg: CadLocalConfig):
|
|
|
138
139
|
const scadPath = join(dir, 'model.scad')
|
|
139
140
|
const stlPath = join(dir, 'model.stl')
|
|
140
141
|
const pngPath = join(dir, 'model.png')
|
|
141
|
-
const
|
|
142
|
-
{
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
142
|
+
const turn = await runBenchRouterTurn(
|
|
143
|
+
{
|
|
144
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
145
|
+
routerKey: cfg.routerKey,
|
|
146
|
+
profile: {
|
|
147
|
+
name: 'cad-local-worker',
|
|
148
|
+
model: { provider: 'tangle-router', default: cfg.model },
|
|
149
|
+
prompt: { systemPrompt: directive },
|
|
150
|
+
},
|
|
151
|
+
},
|
|
152
|
+
user,
|
|
153
|
+
)
|
|
154
|
+
const content = turn.finalText
|
|
155
|
+
if (turn.usage.tokensKnown !== false) {
|
|
156
|
+
usage.input += turn.usage.input
|
|
157
|
+
usage.output += turn.usage.output
|
|
148
158
|
}
|
|
149
159
|
const scad = extractScad(content)
|
|
150
160
|
trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
|
@@ -253,10 +263,19 @@ export async function solveCadRefine(task: BenchTask, cfg: CadRefineConfig): Pro
|
|
|
253
263
|
? task.prompt
|
|
254
264
|
: `Your previous OpenSCAD had this problem:\n${lastErr}\n\nHere is the previous source:\n${history[history.length - 1]?.artifact ?? ''}\n\nFix it so it compiles AND better matches the brief:\n${task.prompt}`,
|
|
255
265
|
runShot: async (user, round, box) => {
|
|
256
|
-
const
|
|
257
|
-
{
|
|
258
|
-
|
|
259
|
-
|
|
266
|
+
const turn = await runBenchRouterTurn(
|
|
267
|
+
{
|
|
268
|
+
routerBaseUrl: cfg.routerBaseUrl,
|
|
269
|
+
routerKey: cfg.routerKey,
|
|
270
|
+
profile: {
|
|
271
|
+
name: 'cad-sandbox-worker',
|
|
272
|
+
model: { provider: 'tangle-router', default: cfg.model },
|
|
273
|
+
prompt: { systemPrompt: sys },
|
|
274
|
+
},
|
|
275
|
+
},
|
|
276
|
+
user,
|
|
277
|
+
)
|
|
278
|
+
const reply = turn.finalText
|
|
260
279
|
const scad = extractScad(reply)
|
|
261
280
|
trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: reply.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
|
262
281
|
trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.scad', content: scad }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)
|
package/src/worker.ts
CHANGED
|
@@ -89,7 +89,7 @@ export async function solveShot(
|
|
|
89
89
|
|
|
90
90
|
// Cold-start-resilient via the shared lineage layer (a gateway-timed-out create is
|
|
91
91
|
// recovered by name lookup). The inline profile + backend override is the same
|
|
92
|
-
// generic AgentRunSpec the
|
|
92
|
+
// generic AgentRunSpec the runAgentRounds kernel boots against the real sandbox.
|
|
93
93
|
const controller = new AbortController()
|
|
94
94
|
const timer = cfg.timeoutMs ? setTimeout(() => controller.abort(), cfg.timeoutMs) : undefined
|
|
95
95
|
const agentRun: AgentRunSpec<string> = {
|
|
@@ -66,6 +66,15 @@ _USAGE_OUT_PATH = f"{_CONTAINER_DIR}/opencode-events.jsonl"
|
|
|
66
66
|
_USAGE_ERR_PATH = f"{_CONTAINER_DIR}/opencode-run.err"
|
|
67
67
|
|
|
68
68
|
|
|
69
|
+
REFINE_DIRECTIVE = (
|
|
70
|
+
"REFINE PASS — your previous attempt failed the task's automated tests. "
|
|
71
|
+
"Use the evidence below to correct the concrete failure without discarding work "
|
|
72
|
+
"that was already right. Verify the task's success criteria before finishing.\n\n"
|
|
73
|
+
"<previous_attempt>\n{prior}\n</previous_attempt>\n\n"
|
|
74
|
+
"--- ORIGINAL TASK ---\n{instruction}"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
69
78
|
class OpenCodeRouterAgent(OpenCodeAgent):
|
|
70
79
|
"""`OpenCodeAgent` + Tangle-router routing + real token metering.
|
|
71
80
|
|
|
@@ -77,27 +86,71 @@ class OpenCodeRouterAgent(OpenCodeAgent):
|
|
|
77
86
|
a real `AgentResult`.
|
|
78
87
|
"""
|
|
79
88
|
|
|
80
|
-
def __init__(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
89
|
+
def __init__(
|
|
90
|
+
self,
|
|
91
|
+
model_name: str,
|
|
92
|
+
profile_path: str,
|
|
93
|
+
prior_attempt_hex: str | None = None,
|
|
94
|
+
*args,
|
|
95
|
+
**kwargs,
|
|
96
|
+
):
|
|
97
|
+
self._profile = json.loads(Path(profile_path).read_text())
|
|
98
|
+
if self._profile.get("harness") != "opencode":
|
|
99
|
+
raise ValueError("AgentProfile.harness must be 'opencode'")
|
|
100
|
+
model = self._profile.get("model")
|
|
101
|
+
if not isinstance(model, dict):
|
|
102
|
+
raise ValueError("AgentProfile.model is required")
|
|
103
|
+
provider = model.get("provider")
|
|
104
|
+
declared = model.get("default")
|
|
105
|
+
if not isinstance(provider, str) or not provider.strip():
|
|
106
|
+
raise ValueError("AgentProfile.model.provider must be explicit")
|
|
107
|
+
if not isinstance(declared, str) or not declared.strip() or declared == "runtime-selected":
|
|
108
|
+
raise ValueError("AgentProfile.model.default must be concrete")
|
|
109
|
+
wire_model = declared if "/" in declared else f"{provider}/{declared}"
|
|
110
|
+
if model_name != wire_model:
|
|
111
|
+
raise ValueError(
|
|
112
|
+
f"Terminal-Bench model {model_name!r} conflicts with AgentProfile {wire_model!r}"
|
|
113
|
+
)
|
|
114
|
+
self._model_name = wire_model
|
|
86
115
|
self._provider = provider
|
|
87
|
-
|
|
88
|
-
self.
|
|
116
|
+
self._provider_model = declared.split("/", 1)[-1]
|
|
117
|
+
self._prior_attempt = self._decode_prior(prior_attempt_hex)
|
|
118
|
+
prompt = self._profile.get("prompt")
|
|
119
|
+
self._profile_instructions: list[str] = []
|
|
120
|
+
if isinstance(prompt, dict):
|
|
121
|
+
system = prompt.get("systemPrompt")
|
|
122
|
+
if isinstance(system, str) and system.strip():
|
|
123
|
+
self._profile_instructions.append(system.strip())
|
|
124
|
+
instructions = prompt.get("instructions")
|
|
125
|
+
if isinstance(instructions, list):
|
|
126
|
+
self._profile_instructions.extend(
|
|
127
|
+
value.strip()
|
|
128
|
+
for value in instructions
|
|
129
|
+
if isinstance(value, str) and value.strip()
|
|
130
|
+
)
|
|
89
131
|
self._router_base_url = (
|
|
90
132
|
os.environ.get("OPENAI_BASE_URL")
|
|
91
133
|
or os.environ.get("ROUTER_BASE_URL")
|
|
92
134
|
or DEFAULT_ROUTER_BASE_URL
|
|
93
135
|
)
|
|
94
136
|
self._router_api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
95
|
-
|
|
96
|
-
# avoid the base split-crash by not delegating provider derivation to it.
|
|
97
|
-
super(OpenCodeAgent, self).__init__(*args, **kwargs)
|
|
98
|
-
self._version = kwargs.get("version", "latest")
|
|
137
|
+
super().__init__(wire_model, *args, **kwargs)
|
|
99
138
|
self._logger = logger.getChild(__name__)
|
|
100
139
|
|
|
140
|
+
@staticmethod
|
|
141
|
+
def _decode_prior(value: str | None) -> str:
|
|
142
|
+
if not value:
|
|
143
|
+
return ""
|
|
144
|
+
raw = str(value).strip()
|
|
145
|
+
if raw.startswith("h"):
|
|
146
|
+
raw = raw[1:]
|
|
147
|
+
if not raw:
|
|
148
|
+
return ""
|
|
149
|
+
try:
|
|
150
|
+
return bytes.fromhex(raw).decode("utf-8").strip()
|
|
151
|
+
except ValueError as error:
|
|
152
|
+
raise ValueError("prior_attempt_hex must be 'h' + hex-encoded UTF-8") from error
|
|
153
|
+
|
|
101
154
|
@staticmethod
|
|
102
155
|
def name() -> str:
|
|
103
156
|
return "agent-runtime-opencode-router"
|
|
@@ -136,16 +189,6 @@ class OpenCodeRouterAgent(OpenCodeAgent):
|
|
|
136
189
|
"OPENAI_BASE_URL": self._router_base_url,
|
|
137
190
|
"OPENCODE_CONFIG": _CONFIG_PATH,
|
|
138
191
|
}
|
|
139
|
-
# Pass through any provider-native key the user already exported, so the same
|
|
140
|
-
# agent also works when pointed at a vendor endpoint rather than the router.
|
|
141
|
-
for passthrough in (
|
|
142
|
-
"ANTHROPIC_API_KEY",
|
|
143
|
-
"DEEPSEEK_API_KEY",
|
|
144
|
-
"GROQ_API_KEY",
|
|
145
|
-
"ZAI_API_KEY",
|
|
146
|
-
):
|
|
147
|
-
if passthrough in os.environ:
|
|
148
|
-
env.setdefault(passthrough, os.environ[passthrough])
|
|
149
192
|
return {k: v for k, v in env.items() if v}
|
|
150
193
|
|
|
151
194
|
def _build_router_config(self) -> str:
|
|
@@ -159,14 +202,7 @@ class OpenCodeRouterAgent(OpenCodeAgent):
|
|
|
159
202
|
"$schema": "https://opencode.ai/config.json",
|
|
160
203
|
# Headless benchmark runs cannot answer interactive permission prompts.
|
|
161
204
|
# Keep this identical for raw and supervisor arms.
|
|
162
|
-
"permission": {
|
|
163
|
-
"edit": "allow",
|
|
164
|
-
"bash": "allow",
|
|
165
|
-
"webfetch": "allow",
|
|
166
|
-
"read": "allow",
|
|
167
|
-
"write": "allow",
|
|
168
|
-
"external_directory": "allow",
|
|
169
|
-
},
|
|
205
|
+
"permission": self._profile.get("permission", {}),
|
|
170
206
|
"provider": {
|
|
171
207
|
self._provider: {
|
|
172
208
|
"npm": "@ai-sdk/openai-compatible",
|
|
@@ -184,25 +220,24 @@ class OpenCodeRouterAgent(OpenCodeAgent):
|
|
|
184
220
|
return json.dumps(config)
|
|
185
221
|
|
|
186
222
|
def _run_agent_commands(self, instruction: str):
|
|
187
|
-
"""
|
|
188
|
-
stream captured to a file so we can meter token usage after the run."""
|
|
223
|
+
"""Instrument Terminal-Bench's stock command without selecting a model here."""
|
|
189
224
|
from terminal_bench.terminal.models import TerminalCommand
|
|
190
225
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
TerminalCommand(
|
|
226
|
+
instrumented = []
|
|
227
|
+
for base in super()._run_agent_commands(instruction):
|
|
228
|
+
marker = " run "
|
|
229
|
+
if marker not in base.command:
|
|
230
|
+
raise ValueError("stock OpenCodeAgent command has no run subcommand")
|
|
231
|
+
command = base.command.replace(marker, " --format json run ", 1)
|
|
232
|
+
command += f" > {_USAGE_OUT_PATH} 2> {_USAGE_ERR_PATH}"
|
|
233
|
+
instrumented.append(TerminalCommand(
|
|
199
234
|
command=command,
|
|
200
235
|
min_timeout_sec=0.0,
|
|
201
236
|
max_timeout_sec=float("inf"),
|
|
202
237
|
block=True,
|
|
203
238
|
append_enter=True,
|
|
204
|
-
)
|
|
205
|
-
|
|
239
|
+
))
|
|
240
|
+
return instrumented
|
|
206
241
|
|
|
207
242
|
def _write_config_to_container(self, session: TmuxSession) -> None:
|
|
208
243
|
config_json = self._build_router_config()
|
|
@@ -365,6 +400,13 @@ class OpenCodeRouterAgent(OpenCodeAgent):
|
|
|
365
400
|
"OPENAI_API_KEY (router key) is not set; opencode has no credential "
|
|
366
401
|
"to reach the router."
|
|
367
402
|
)
|
|
403
|
+
if self._profile_instructions:
|
|
404
|
+
instruction = "\n\n".join([*self._profile_instructions, instruction])
|
|
405
|
+
if self._prior_attempt:
|
|
406
|
+
instruction = REFINE_DIRECTIVE.format(
|
|
407
|
+
prior=self._prior_attempt,
|
|
408
|
+
instruction=instruction,
|
|
409
|
+
)
|
|
368
410
|
# Publish container id + inject router config BEFORE the base flow runs the
|
|
369
411
|
# agent. The base perform_task copies the install script into _CONTAINER_DIR,
|
|
370
412
|
# sources setup-env.sh (which exports OPENCODE_CONFIG), installs opencode, and
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cadbench-BrpwOU6A.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n try {\n const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n return { code: 0, stdout, stderr }\n } catch (err) {\n const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n for c in o.bound_box:\n w=o.matrix_world @ mathutils.Vector(c)\n for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n az=math.radians(40 + v*360.0/nviews)\n cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n * Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n const views = Math.max(1, opts.views ?? 4)\n const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n try {\n await writeFile(runnerPath, RUNNER_PY)\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n const built = /RENDER_DONE/.test(out)\n if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n const renders: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n return { built: renders.length > 0, renders }\n } finally {\n await rm(dir, { recursive: true, force: true }).catch(() => {})\n }\n}\n\nexport interface BlenderLocalConfig {\n routerBaseUrl: string\n routerKey: string\n model: string\n rounds?: number\n /** N standardized views to render (CADBench uses 4). Default 4. */\n views?: number\n /** The bpy authoring directive — the GEPA-optimizable surface. */\n directive?: string\n}\n\nexport interface BlenderShot {\n /** The bpy script the agent wrote — the artifact. */\n artifact: string\n /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n renders: string[]\n trace: Span[]\n usage: { input: number; output: number }\n ok: boolean\n /** True if the script executed and produced at least one mesh. */\n built: boolean\n detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n const rounds = Math.max(1, cfg.rounds ?? 2)\n const views = Math.max(1, cfg.views ?? 4)\n const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n const trace: Span[] = []\n const runId = `cadbench-${task.id}`\n let ts = Date.now()\n const tick = () => (ts += 1)\n const usage = { input: 0, output: 0 }\n // Carried across rounds in closures (the round Artifact is the bpy script; the\n // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n let renders: string[] = []\n let built = false\n let lastErr = ''\n\n trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n // lastErr + the prior script verbatim.\n const res = await runRefineLoop<string, string>({\n rounds,\n setup: async () => {\n const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n return dir\n },\n prompt: (round, history) =>\n round === 1\n ? task.prompt\n : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n runShot: async (user, round, dir) => {\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n const { content, usage: u } = await routerChatWithUsage(\n cfg,\n [\n { role: 'system', content: directive },\n { role: 'user', content: user },\n ],\n { temperature: 0.3 },\n )\n if (u) {\n usage.input += u.input\n usage.output += u.output\n }\n const script = extractPy(content)\n trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n built = /RENDER_DONE/.test(out)\n lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n if (built) {\n const collected: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n renders = collected\n // first view carries the screen span (run-capsule reveal)\n trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n built = collected.length > 0\n }\n return { artifact: script }\n },\n judge: async () => ({ valid: built }),\n teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n })\n\n const script = res.final.artifact\n return {\n artifact: script,\n renders,\n trace,\n usage,\n ok: script.trim().length > 0,\n built,\n detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n name: string\n type: string\n criteria: string[]\n}\n\nfunction must(name: string): string {\n const v = process.env[name]\n if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n * criteria → a JSON array of booleans (true = satisfied). Faithful to the\n * paper's combined image+script evaluation. Throws on transport failure (never\n * a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n instruction: string,\n script: string,\n criteria: string[],\n renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n const key = must('TANGLE_API_KEY')\n const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n const text =\n `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n const content: unknown[] = [{ type: 'text', text }]\n for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n const res = await fetch(`${base}/chat/completions`, {\n method: 'POST',\n headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n })\n if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n const raw = data.choices?.[0]?.message?.content ?? ''\n const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n let arr: unknown\n try {\n arr = JSON.parse(m[0].toLowerCase())\n } catch {\n return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n }\n const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n // Pad/truncate to criteria length (a short array scores the missing as fail).\n const passed = criteria.map((_, i) => bools[i] === true)\n return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n async function load(): Promise<typeof cache & object> {\n if (cache) return cache\n const path = process.env.CADBENCH_PATH\n if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n const text = await readFile(path, 'utf8')\n cache = text\n .split('\\n')\n .filter((l) => l.trim())\n .map((l) => {\n const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n })\n return cache\n }\n\n return {\n name: 'cadbench',\n\n async preflight() {\n const { execFile } = await import('node:child_process')\n const { promisify } = await import('node:util')\n const exec = promisify(execFile)\n try {\n await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n } catch (err) {\n throw new Error(\n `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n )\n }\n await load()\n },\n\n async loadTasks(opts: LoadOptions = {}) {\n let rows = await load()\n if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n // TYPE filter (Simulative|Wild) via env, applied before limit.\n const t = process.env.CADBENCH_TYPE\n if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n if (opts.limit != null) rows = rows.slice(0, opts.limit)\n return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n },\n\n async goldArtifact() {\n return undefined // no reference bpy script ships with the benchmark\n },\n\n async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n const meta = task.metadata as unknown as CadBenchMeta\n const criteria = meta.criteria ?? []\n if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n const r = await renderBpy(artifact, { views: 4 })\n if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n const score = passed.filter(Boolean).length / criteria.length\n return { resolved: score === 1, score, detail: note }\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;AC1FA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CACjF,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,oBAAoB;EAClD,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU;EAAM;EAC9E,MAAM,KAAK,UAAU;GAAE;GAAO,YAAY;GAAM,aAAa;GAAG,UAAU,CAAC;IAAE,MAAM;IAAQ;GAAQ,CAAC;EAAE,CAAC;CACzG,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAEhG,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,UAAU,EAAE,EAAE,SAAS,WAAW;CACnD,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}
|