@tangle-network/agent-bench 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/HARNESS.md +1 -1
- package/dist/adapters.js +2 -2
- package/dist/benchmarks/agentbench.d.ts +1 -1
- package/dist/benchmarks/agentbench.js.map +1 -1
- package/dist/benchmarks/appworld.d.ts +1 -1
- package/dist/benchmarks/appworld.js +1 -1
- package/dist/benchmarks/appworld.js.map +1 -1
- package/dist/benchmarks/bfcl.d.ts +1 -1
- package/dist/benchmarks/bfcl.js.map +1 -1
- package/dist/benchmarks/cadbench.js +1 -1
- package/dist/benchmarks/cadgenbench.js +1 -1
- package/dist/benchmarks/commit0.d.ts +1 -1
- package/dist/benchmarks/commit0.js.map +1 -1
- package/dist/benchmarks/dabstep.d.ts +1 -1
- package/dist/benchmarks/dabstep.js.map +1 -1
- package/dist/benchmarks/enterpriseops-gym.d.ts +1 -1
- package/dist/benchmarks/enterpriseops-gym.js.map +1 -1
- package/dist/benchmarks/programbench.d.ts +1 -1
- package/dist/benchmarks/programbench.js.map +1 -1
- package/dist/benchmarks/rag-shared.d.ts +1 -1
- package/dist/benchmarks/rag-shared.js.map +1 -1
- package/dist/benchmarks/swe-bench.d.ts +1 -1
- package/dist/benchmarks/swe-bench.js.map +1 -1
- package/dist/benchmarks/tau-bench-shared.d.ts +1 -1
- package/dist/benchmarks/tau-bench-shared.js.map +1 -1
- package/dist/benchmarks/tau2-bench.d.ts +1 -1
- package/dist/benchmarks/toollm.d.ts +1 -1
- package/dist/benchmarks/toollm.js.map +1 -1
- package/dist/benchmarks/types.d.ts +1 -1
- package/dist/benchmarks/webarena-verified.d.ts +1 -1
- package/dist/benchmarks/webarena-verified.js.map +1 -1
- package/dist/{cadbench-DpQWZHp4.js → cadbench-BrpwOU6A.js} +2 -2
- package/dist/cadbench-BrpwOU6A.js.map +1 -0
- package/dist/{cadgenbench-DRhczfsG.js → cadgenbench-DF7hYHdl.js} +2 -2
- package/dist/cadgenbench-DF7hYHdl.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
- package/scripts/verify-packed-consumer.mjs +38 -15
- package/scripts/wait-for-published-dependencies.mjs +245 -0
- package/scripts/wait-for-published-dependencies.test.mjs +96 -0
- package/src/aec-gate.mts +1 -1
- package/src/benchmarks/agentbench.ts +1 -1
- package/src/benchmarks/appworld.ts +1 -1
- package/src/benchmarks/bfcl.ts +1 -1
- package/src/benchmarks/commit0.ts +1 -1
- package/src/benchmarks/dabstep.ts +1 -1
- package/src/benchmarks/enterpriseops-gym.ts +1 -1
- package/src/benchmarks/programbench.ts +1 -1
- package/src/benchmarks/rag-shared.ts +1 -1
- package/src/benchmarks/swe-bench.ts +1 -1
- package/src/benchmarks/tau-bench-shared.ts +1 -1
- package/src/benchmarks/toollm.ts +1 -1
- package/src/benchmarks/types.ts +1 -1
- package/src/benchmarks/webarena-verified.ts +1 -1
- package/src/clbench-codebase-gate.mts +1 -1
- package/src/clbench-context-gate.mts +1 -1
- package/src/cloud-loop.mts +1 -1
- package/src/commit0-env-run.mts +1 -1
- package/src/commit0-env.ts +1 -1
- package/src/commit0-gate.mts +1 -1
- package/src/corpus.ts +1 -1
- package/src/examples/lean-proof-gate.mts +1 -1
- package/src/examples/math-demo.mts +1 -1
- package/src/examples/strategy-demo.mts +1 -1
- package/src/fleet.mts +1 -1
- package/src/gate-cli.mts +1 -1
- package/src/gate.test.mts +1 -1
- package/src/gate.ts +2 -2
- package/src/generate-eval/certify.ts +1 -1
- package/src/humaneval-gate.mts +1 -1
- package/src/humaneval-repair-gate.mts +1 -1
- package/src/research-shot.ts +1 -1
- package/src/resolve-client.ts +1 -1
- package/src/router-executor.ts +1 -1
- package/src/run-benchmarks-report.ts +1 -1
- package/src/run-benchmarks.test.mts +1 -1
- package/src/run-benchmarks.ts +2 -2
- package/src/sandbox-run.ts +3 -3
- package/src/search-bench/bridge.ts +1 -1
- package/src/search-bench/parametric-check.mts +1 -1
- package/src/search-bench/run.mts +1 -1
- package/src/search-tool.ts +1 -1
- package/src/swe-arena/fixtures/factory/loops-28/calibration.md +1 -1
- package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +1 -1
- package/src/swe-bench-env.ts +1 -1
- package/src/swe-emit-patch.mts +2 -2
- package/src/swe-improve.mts +2 -2
- package/src/swe-local-proof.mts +2 -2
- package/src/swe-repro-calibrate.mts +1 -1
- package/src/swe-self-improve.mts +1 -1
- package/src/swe-stream.mts +2 -2
- package/src/swe-structural.mts +2 -2
- package/src/tb-container-executor.test.mts +1 -1
- package/src/worker-blender.ts +1 -1
- package/src/worker-browser.ts +1 -1
- package/src/worker-build123d.ts +1 -1
- package/src/worker-cad.ts +1 -1
- package/src/worker.ts +1 -1
- package/dist/cadbench-DpQWZHp4.js.map +0 -1
- package/dist/cadgenbench-DRhczfsG.js.map +0 -1
package/src/corpus.ts
CHANGED
|
@@ -16,7 +16,7 @@ import { appendFile, mkdir } from 'node:fs/promises'
|
|
|
16
16
|
import { dirname } from 'node:path'
|
|
17
17
|
import { hashContent, type RunSplitTag, validateRunRecord } from '@tangle-network/agent-eval'
|
|
18
18
|
import type { CorpusRecord } from '@tangle-network/agent-eval/rl'
|
|
19
|
-
import type { Iteration } from '@tangle-network/agent-runtime/
|
|
19
|
+
import type { Iteration } from '@tangle-network/agent-runtime/kernel'
|
|
20
20
|
import type { BenchRuntimeDecisionPoint, BenchRuntimeHookEvent } from './runtime-hook-recorder'
|
|
21
21
|
|
|
22
22
|
/** One attempt within a condition-run: the prompt/steer sent, the output, the
|
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
runBenchmark,
|
|
23
23
|
sample,
|
|
24
24
|
sampleThenRefine,
|
|
25
|
-
} from '@tangle-network/agent-runtime/
|
|
25
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
26
26
|
import { ensureLeanImage, leanCheck } from './lean-verify.js'
|
|
27
27
|
|
|
28
28
|
// Real, mathlib-free Lean 4 theorems (compile in core Lean). `header` is everything up to `:=`;
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
runBenchmark,
|
|
20
20
|
sample,
|
|
21
21
|
sampleThenRefine,
|
|
22
|
-
} from '@tangle-network/agent-runtime/
|
|
22
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
23
23
|
|
|
24
24
|
// GSM8K-style problems; meta.answer is the ground truth the deployable check compares to.
|
|
25
25
|
const problems: Array<{ q: string; answer: number }> = [
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* Toy task = only a router key needed (no dataset, no sandbox). Run from bench/:
|
|
13
13
|
* TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
|
|
14
14
|
*/
|
|
15
|
-
import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/
|
|
15
|
+
import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/kernel'
|
|
16
16
|
|
|
17
17
|
// ── 1. Implement an Environment (the only thing a new domain writes) ──────────────
|
|
18
18
|
// A toy: the agent must drive a counter to exactly the target using the increment tool.
|
package/src/fleet.mts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* Run it twice: the second run injects the first run's learnings into the workers.
|
|
14
14
|
*/
|
|
15
15
|
import { createChatClient } from '@tangle-network/agent-eval'
|
|
16
|
-
import { FileCorpus, observe, openSandboxRun, renderReport } from '@tangle-network/agent-runtime/
|
|
16
|
+
import { FileCorpus, observe, openSandboxRun, renderReport } from '@tangle-network/agent-runtime/kernel'
|
|
17
17
|
import { Sandbox } from '@tangle-network/sandbox'
|
|
18
18
|
import { answerOutput, sandboxAgentRun, type WorkerBackendType } from './sandbox-run'
|
|
19
19
|
|
package/src/gate-cli.mts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* single CLI run is the instrument, not the significance test).
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type { AgentProfile } from '@tangle-network/agent-runtime/
|
|
18
|
+
import type { AgentProfile } from '@tangle-network/agent-runtime/kernel'
|
|
19
19
|
import { resolveAdapter } from './adapters'
|
|
20
20
|
import { runGate } from './gate'
|
|
21
21
|
|
package/src/gate.test.mts
CHANGED
|
@@ -19,7 +19,7 @@ import type {
|
|
|
19
19
|
Executor,
|
|
20
20
|
ExecutorFactory,
|
|
21
21
|
ExecutorResult,
|
|
22
|
-
} from '@tangle-network/agent-runtime/
|
|
22
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
23
23
|
import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
|
|
24
24
|
import { runGate, type SolveTask } from './gate'
|
|
25
25
|
|
package/src/gate.ts
CHANGED
|
@@ -46,7 +46,7 @@ import type {
|
|
|
46
46
|
Spend,
|
|
47
47
|
SupervisedResult,
|
|
48
48
|
TrajectoryReport,
|
|
49
|
-
} from '@tangle-network/agent-runtime/
|
|
49
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
50
50
|
import {
|
|
51
51
|
definePersona,
|
|
52
52
|
routerChatWithUsage,
|
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
InMemorySpawnJournal,
|
|
57
57
|
runPersonified,
|
|
58
58
|
trajectoryReport,
|
|
59
|
-
} from '@tangle-network/agent-runtime/
|
|
59
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
60
60
|
import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
|
|
61
61
|
|
|
62
62
|
/** A fanout child's task: the prompt to solve with + the instance to grade against. The instance
|
|
@@ -24,7 +24,7 @@ import { execSync } from 'node:child_process'
|
|
|
24
24
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
25
25
|
import { tmpdir } from 'node:os'
|
|
26
26
|
import { dirname, join, resolve } from 'node:path'
|
|
27
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
27
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
28
28
|
import { scoreTask, taskToPrompt } from '../search-bench/tasks'
|
|
29
29
|
import { type EvalCertification, type GeneratedEval, generatedEvalSchemaVersion, parseCandidate } from './schema'
|
|
30
30
|
|
package/src/humaneval-gate.mts
CHANGED
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
|
|
44
44
|
import { composeStrategies } from './directives'
|
|
45
45
|
import { basePrompt, type CheckResult, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
|
|
46
|
-
import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
46
|
+
import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
47
47
|
import { selfConsistencySelect, verifierGroundedSelect } from './selector'
|
|
48
48
|
import { type PairedLift, pairedLift, pool } from './stats.mts'
|
|
49
49
|
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* tsx src/humaneval-repair-gate.mts
|
|
21
21
|
*/
|
|
22
22
|
import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval, runChecker } from './benchmarks/humaneval'
|
|
23
|
-
import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/
|
|
23
|
+
import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/kernel'
|
|
24
24
|
import { verifierGroundedSelect } from './selector'
|
|
25
25
|
import { type PairedLift, pairedLift, pool } from './stats.mts'
|
|
26
26
|
|
package/src/research-shot.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* — the only difference is who drives the rounds (a flat best-of-k pool vs the real
|
|
11
11
|
* `runLoop` kernel with analyst steering).
|
|
12
12
|
*/
|
|
13
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
13
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
14
14
|
|
|
15
15
|
export interface ShotCfg {
|
|
16
16
|
model: string
|
package/src/resolve-client.ts
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
inlineSandboxClient,
|
|
25
25
|
resolveSandboxClient,
|
|
26
26
|
type SandboxClient,
|
|
27
|
-
} from '@tangle-network/agent-runtime/
|
|
27
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
28
28
|
import { Sandbox } from '@tangle-network/sandbox'
|
|
29
29
|
import { makeSearchExecutor, webSearchTool } from './search-tool'
|
|
30
30
|
|
package/src/router-executor.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
type ExecutorResult,
|
|
21
21
|
inlineSandboxClient,
|
|
22
22
|
type SandboxClient,
|
|
23
|
-
} from '@tangle-network/agent-runtime/
|
|
23
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
24
24
|
import { runResearchShot, type ShotCfg } from './research-shot'
|
|
25
25
|
|
|
26
26
|
export function routerSandboxClient(cfg: ShotCfg): SandboxClient {
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
renderLeaderboardHtml,
|
|
17
17
|
renderLeaderboardMarkdown,
|
|
18
18
|
renderLeaderboardSvg,
|
|
19
|
-
} from '@tangle-network/agent-runtime/
|
|
19
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
20
20
|
import type { BenchCellTaskResult, RunBenchmarksReport } from './run-benchmarks'
|
|
21
21
|
|
|
22
22
|
/** Project the runner's per-task results into `RunRecord`s the leaderboard engine reads. Cost/tokens are
|
|
@@ -137,7 +137,7 @@ async function main(): Promise<void> {
|
|
|
137
137
|
assert.equal(leakyPrompts[1]!.includes(leakyGold), false, 'retry prompt redacts hidden gold fields')
|
|
138
138
|
assert.match(leakyPrompts[1]!, /publicHint/)
|
|
139
139
|
|
|
140
|
-
const runtime = await import('@tangle-network/agent-runtime/
|
|
140
|
+
const runtime = await import('@tangle-network/agent-runtime/kernel')
|
|
141
141
|
if (runtime.openSandboxRun.toString().includes('beforeStart')) {
|
|
142
142
|
// The default shot path supports benchmark-owned box setup/extract without real sandbox infra.
|
|
143
143
|
const order: string[] = []
|
package/src/run-benchmarks.ts
CHANGED
|
@@ -34,8 +34,8 @@ import type {
|
|
|
34
34
|
AgentRunSpec,
|
|
35
35
|
Deliverable,
|
|
36
36
|
OpenSandboxRunOptions,
|
|
37
|
-
} from '@tangle-network/agent-runtime/
|
|
38
|
-
import { openSandboxRun } from '@tangle-network/agent-runtime/
|
|
37
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
38
|
+
import { openSandboxRun } from '@tangle-network/agent-runtime/kernel'
|
|
39
39
|
import type { SandboxEvent } from '@tangle-network/sandbox'
|
|
40
40
|
import { resolveAdapter } from './adapters'
|
|
41
41
|
import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
|
package/src/sandbox-run.ts
CHANGED
|
@@ -14,9 +14,9 @@ import {
|
|
|
14
14
|
type AgentRunSpec,
|
|
15
15
|
type OutputAdapter,
|
|
16
16
|
routerChatWithUsage,
|
|
17
|
-
} from '@tangle-network/agent-runtime/
|
|
18
|
-
// `BackendType` is the sandbox SDK's harness union
|
|
19
|
-
// it from there too; it is not re-exported from the
|
|
17
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
18
|
+
// `BackendType` is the sandbox SDK's harness union and its canonical home. agent-runtime consumes
|
|
19
|
+
// it from there too; it is not re-exported from the kernel barrel.
|
|
20
20
|
import type { BackendType } from '@tangle-network/sandbox'
|
|
21
21
|
|
|
22
22
|
/** Parse the agent's final answer from the event stream (harness-agnostic).
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* The bridge model id IS the harness selector (e.g. `claude-code/sonnet`,
|
|
13
13
|
* `opencode/zai-coding-plan/glm-5.1`), so `harness` here is just the label.
|
|
14
14
|
*/
|
|
15
|
-
import { createExecutor } from '@tangle-network/agent-runtime/
|
|
15
|
+
import { createExecutor } from '@tangle-network/agent-runtime/kernel'
|
|
16
16
|
import type { SearchArm } from './profiles'
|
|
17
17
|
import { armLabel } from './profiles'
|
|
18
18
|
import type { SearchCellResult } from './run.mts'
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { writeFileSync } from 'node:fs'
|
|
13
13
|
import { runPool } from '../run-pool'
|
|
14
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
14
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
15
15
|
import { freshTasks } from './tasks-fresh'
|
|
16
16
|
import { scoreTask, taskToPrompt } from './tasks'
|
|
17
17
|
|
package/src/search-bench/run.mts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
|
|
17
17
|
import { dirname } from 'node:path'
|
|
18
|
-
import { extractLlmCallEvent, openSandboxRun } from '@tangle-network/agent-runtime/
|
|
18
|
+
import { extractLlmCallEvent, openSandboxRun } from '@tangle-network/agent-runtime/kernel'
|
|
19
19
|
import { Sandbox, type SandboxEvent } from '@tangle-network/sandbox'
|
|
20
20
|
import { answerOutput, sandboxAgentRun, type WorkerBackendType } from '../sandbox-run'
|
|
21
21
|
import { type BridgeCfg, runBridgeCell } from './bridge'
|
package/src/search-tool.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* a recall-only answer; a valid-but-empty result returns an explicit "no results"
|
|
14
14
|
* string the model can reason about.
|
|
15
15
|
*/
|
|
16
|
-
import type { ToolSpec } from '@tangle-network/agent-runtime/
|
|
16
|
+
import type { ToolSpec } from '@tangle-network/agent-runtime/kernel'
|
|
17
17
|
|
|
18
18
|
export const webSearchTool: ToolSpec = {
|
|
19
19
|
type: 'function',
|
|
@@ -43,5 +43,5 @@ All three fail at collection: `src/best-effort.ts`, `src/worker-clone.ts`, `src/
|
|
|
43
43
|
- Tests spawn `git` in temporary directories with identity passed inline, and use symlink/chmod, so they are POSIX-only. The judge runs in the pinned factory container with network disabled and no operator environment. **Excluded: none.**
|
|
44
44
|
- `tests/worker-evidence.test.ts` imports the bound constants (`EVIDENCE_MAX_CHARS` etc.) and asserts **relative to them** (`length ≤ EVIDENCE_MAX_CHARS`, tail contains `'v'.repeat(VERIFY_TAIL_CHARS)`) — exact values are the builder's choice; the names and semantics are contract and are in `spec.md`. Not trimmed: self-referencing bounds have behavioral content (bounding + tail retention).
|
|
45
45
|
- One pinned prose contract: the best-effort delivery commit subject must contain `best-effort delivery from <label>` and `no worker passed the verify gate` — promoted into `spec.md` as acceptance criteria.
|
|
46
|
-
- Judge imports `@tangle-network/agent-runtime/
|
|
46
|
+
- Judge imports `@tangle-network/agent-runtime/kernel` (`gitWorkspace`, `runInWorkspace`) — a published npm dependency already in the base lockfile, not sibling unmerged work.
|
|
47
47
|
- Scope note: the merged PR also tuned worker budgets and prompts (`extensions/pi/`, `src/top-model.ts`); the judge does not cover those, so this instance grades the three library modules only. The spec describes the modules as the deliverable.
|
|
@@ -71,7 +71,7 @@ index e83e6d9..3529d79 100644
|
|
|
71
71
|
--- a/extensions/pi/loops.ts
|
|
72
72
|
+++ b/extensions/pi/loops.ts
|
|
73
73
|
@@ -42,7 +42,7 @@ import {
|
|
74
|
-
} from '@tangle-network/agent-runtime/
|
|
74
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
75
75
|
import { resumeStrategyLoop } from '../../src/runner.js'
|
|
76
76
|
import { cancelStrategyLoop, listStrategyLoops, renderStrategyLoop, startStrategyLoop } from '../../src/strategy-loop.js'
|
|
77
77
|
-import { type BestEffortDelivery, deliverBestEffortPatch } from '../../src/best-effort.js'
|
package/src/swe-bench-env.ts
CHANGED
|
@@ -16,7 +16,7 @@ import { execFile } from 'node:child_process'
|
|
|
16
16
|
import { cpSync, existsSync, lstatSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'
|
|
17
17
|
import { join, sep } from 'node:path'
|
|
18
18
|
import { promisify } from 'node:util'
|
|
19
|
-
import type { AgenticSurface, AgenticTask, AgenticTool, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
19
|
+
import type { AgenticSurface, AgenticTask, AgenticTool, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
20
20
|
import { runVenvPython } from './benchmarks/_harness'
|
|
21
21
|
import { createSweBenchAdapter, type SweBenchAdapterOptions } from './benchmarks/swe-bench'
|
|
22
22
|
import type { BenchTask } from './benchmarks/types'
|
package/src/swe-emit-patch.mts
CHANGED
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
*/
|
|
23
23
|
import { execFile } from 'node:child_process'
|
|
24
24
|
import { promisify } from 'node:util'
|
|
25
|
-
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
26
|
-
import { refine, runAgentic } from '@tangle-network/agent-runtime/
|
|
25
|
+
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
26
|
+
import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel'
|
|
27
27
|
import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env'
|
|
28
28
|
|
|
29
29
|
const exec = promisify(execFile)
|
package/src/swe-improve.mts
CHANGED
|
@@ -32,8 +32,8 @@ import {
|
|
|
32
32
|
canonicalCandidateDigest,
|
|
33
33
|
type AgentProfile,
|
|
34
34
|
} from '@tangle-network/agent-interface'
|
|
35
|
-
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
36
|
-
import { refine, runAgentic } from '@tangle-network/agent-runtime/
|
|
35
|
+
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
36
|
+
import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel'
|
|
37
37
|
import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
|
|
38
38
|
import { createSweBenchAdapter } from './benchmarks/swe-bench'
|
|
39
39
|
import type { BenchTask } from './benchmarks/types'
|
package/src/swe-local-proof.mts
CHANGED
|
@@ -20,8 +20,8 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
|
20
20
|
import { tmpdir } from 'node:os'
|
|
21
21
|
import { join } from 'node:path'
|
|
22
22
|
import { promisify } from 'node:util'
|
|
23
|
-
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
24
|
-
import { refine, runAgentic } from '@tangle-network/agent-runtime/
|
|
23
|
+
import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
24
|
+
import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel'
|
|
25
25
|
import type { BenchScore } from './benchmarks/types'
|
|
26
26
|
import { createSweBenchEnvironment } from './swe-bench-env'
|
|
27
27
|
|
|
@@ -27,7 +27,7 @@ import { appendFileSync, cpSync, mkdtempSync, rmSync, writeFileSync } from 'node
|
|
|
27
27
|
import { tmpdir } from 'node:os'
|
|
28
28
|
import { join } from 'node:path'
|
|
29
29
|
import { promisify } from 'node:util'
|
|
30
|
-
import type { AgenticTask, ArtifactHandle } from '@tangle-network/agent-runtime/
|
|
30
|
+
import type { AgenticTask, ArtifactHandle } from '@tangle-network/agent-runtime/kernel'
|
|
31
31
|
import type { BenchTask } from './benchmarks/types'
|
|
32
32
|
import { createSweBenchEnvironment, resolveImageForMetadata } from './swe-bench-env'
|
|
33
33
|
import {
|
package/src/swe-self-improve.mts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { mkdtempSync, rmSync } from 'node:fs'
|
|
11
11
|
import { join } from 'node:path'
|
|
12
12
|
import { createChatClient } from '@tangle-network/agent-eval'
|
|
13
|
-
import { refine, runAgentic, runStrategyEvolution, sample } from '@tangle-network/agent-runtime/
|
|
13
|
+
import { refine, runAgentic, runStrategyEvolution, sample } from '@tangle-network/agent-runtime/kernel'
|
|
14
14
|
import { createSweBenchEnvironment } from './swe-bench-env'
|
|
15
15
|
|
|
16
16
|
async function main(): Promise<void> {
|
package/src/swe-stream.mts
CHANGED
|
@@ -65,8 +65,8 @@ import { homedir } from 'node:os'
|
|
|
65
65
|
import { join } from 'node:path'
|
|
66
66
|
import { promisify } from 'node:util'
|
|
67
67
|
import { applyKnowledgeWriteBlocks, buildKnowledgeIndex, initKnowledgeBase, searchKnowledge } from '@tangle-network/agent-knowledge'
|
|
68
|
-
import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
69
|
-
import { refine, runAgentic } from '@tangle-network/agent-runtime/
|
|
68
|
+
import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
69
|
+
import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel'
|
|
70
70
|
import type { BenchTask } from './benchmarks/types'
|
|
71
71
|
import { createSweBenchEnvironment, resolveImageForMetadata, SWE_SEED_PROMPT } from './swe-bench-env'
|
|
72
72
|
import {
|
package/src/swe-structural.mts
CHANGED
|
@@ -49,8 +49,8 @@ import { execFile } from 'node:child_process'
|
|
|
49
49
|
import { appendFileSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
50
50
|
import { join } from 'node:path'
|
|
51
51
|
import { promisify } from 'node:util'
|
|
52
|
-
import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/
|
|
53
|
-
import { refine, runAgentic } from '@tangle-network/agent-runtime/
|
|
52
|
+
import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel'
|
|
53
|
+
import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel'
|
|
54
54
|
import type { BenchTask } from './benchmarks/types'
|
|
55
55
|
import { createSweBenchEnvironment, resolveImageForMetadata, SWE_SEED_PROMPT } from './swe-bench-env'
|
|
56
56
|
import {
|
package/src/worker-blender.ts
CHANGED
|
@@ -18,7 +18,7 @@ import type { Span } from '@tangle-network/agent-eval'
|
|
|
18
18
|
import type { BenchTask } from './benchmarks/types'
|
|
19
19
|
import { DEFAULT_BLENDER_DIRECTIVE } from './directives'
|
|
20
20
|
import { runRefineLoop } from './refine-loop'
|
|
21
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
21
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
22
22
|
|
|
23
23
|
export { DEFAULT_BLENDER_DIRECTIVE } from './directives'
|
|
24
24
|
|
package/src/worker-browser.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import { readFile } from 'node:fs/promises'
|
|
19
19
|
import type { Span } from '@tangle-network/agent-eval'
|
|
20
20
|
import type { BenchTask } from './benchmarks/types'
|
|
21
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
21
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
22
22
|
|
|
23
23
|
export interface BrowserLocalConfig {
|
|
24
24
|
routerBaseUrl: string
|
package/src/worker-build123d.ts
CHANGED
|
@@ -21,7 +21,7 @@ import type { Span } from '@tangle-network/agent-eval'
|
|
|
21
21
|
import type { BenchTask } from './benchmarks/types'
|
|
22
22
|
import { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
|
|
23
23
|
import { runRefineLoop } from './refine-loop'
|
|
24
|
-
import { routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
24
|
+
import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
25
25
|
|
|
26
26
|
export { DEFAULT_BUILD123D_DIRECTIVE } from './directives'
|
|
27
27
|
|
package/src/worker-cad.ts
CHANGED
|
@@ -17,7 +17,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
|
17
17
|
import { tmpdir } from 'node:os'
|
|
18
18
|
import { join } from 'node:path'
|
|
19
19
|
import { promisify } from 'node:util'
|
|
20
|
-
import { acquireSandbox, routerChatWithUsage } from '@tangle-network/agent-runtime/
|
|
20
|
+
import { acquireSandbox, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
|
|
21
21
|
import { Sandbox } from '@tangle-network/sandbox'
|
|
22
22
|
import type { Span } from '@tangle-network/agent-eval'
|
|
23
23
|
import type { BenchTask } from './benchmarks/types'
|
package/src/worker.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
type AgentRunSpec,
|
|
11
11
|
type Deliverable,
|
|
12
12
|
openSandboxRun,
|
|
13
|
-
} from '@tangle-network/agent-runtime/
|
|
13
|
+
} from '@tangle-network/agent-runtime/kernel'
|
|
14
14
|
import { Sandbox } from '@tangle-network/sandbox'
|
|
15
15
|
import type { BenchTask } from './benchmarks/types'
|
|
16
16
|
import {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cadbench-DpQWZHp4.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n try {\n const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n return { code: 0, stdout, stderr }\n } catch (err) {\n const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n for c in o.bound_box:\n w=o.matrix_world @ mathutils.Vector(c)\n for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n az=math.radians(40 + v*360.0/nviews)\n cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n * Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n const views = Math.max(1, opts.views ?? 4)\n const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n try {\n await writeFile(runnerPath, RUNNER_PY)\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n const built = /RENDER_DONE/.test(out)\n if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n const renders: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n return { built: renders.length > 0, renders }\n } finally {\n await rm(dir, { recursive: true, force: true }).catch(() => {})\n }\n}\n\nexport interface BlenderLocalConfig {\n routerBaseUrl: string\n routerKey: string\n model: string\n rounds?: number\n /** N standardized views to render (CADBench uses 4). Default 4. */\n views?: number\n /** The bpy authoring directive — the GEPA-optimizable surface. */\n directive?: string\n}\n\nexport interface BlenderShot {\n /** The bpy script the agent wrote — the artifact. */\n artifact: string\n /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n renders: string[]\n trace: Span[]\n usage: { input: number; output: number }\n ok: boolean\n /** True if the script executed and produced at least one mesh. */\n built: boolean\n detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n const rounds = Math.max(1, cfg.rounds ?? 2)\n const views = Math.max(1, cfg.views ?? 4)\n const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n const trace: Span[] = []\n const runId = `cadbench-${task.id}`\n let ts = Date.now()\n const tick = () => (ts += 1)\n const usage = { input: 0, output: 0 }\n // Carried across rounds in closures (the round Artifact is the bpy script; the\n // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n let renders: string[] = []\n let built = false\n let lastErr = ''\n\n trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n // lastErr + the prior script verbatim.\n const res = await runRefineLoop<string, string>({\n rounds,\n setup: async () => {\n const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n return dir\n },\n prompt: (round, history) =>\n round === 1\n ? task.prompt\n : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n runShot: async (user, round, dir) => {\n const runnerPath = join(dir, 'runner.py')\n const scriptPath = join(dir, 'model.py')\n const { content, usage: u } = await routerChatWithUsage(\n cfg,\n [\n { role: 'system', content: directive },\n { role: 'user', content: user },\n ],\n { temperature: 0.3 },\n )\n if (u) {\n usage.input += u.input\n usage.output += u.output\n }\n const script = extractPy(content)\n trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n await writeFile(scriptPath, script)\n const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n const out = `${run.stdout}\\n${run.stderr}`\n built = /RENDER_DONE/.test(out)\n lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n if (built) {\n const collected: string[] = []\n for (let v = 0; v < views; v++) {\n const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n }\n renders = collected\n // first view carries the screen span (run-capsule reveal)\n trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n built = collected.length > 0\n }\n return { artifact: script }\n },\n judge: async () => ({ valid: built }),\n teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n })\n\n const script = res.final.artifact\n return {\n artifact: script,\n renders,\n trace,\n usage,\n ok: script.trim().length > 0,\n built,\n detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\n\ninterface CadBenchMeta {\n name: string\n type: string\n criteria: string[]\n}\n\nfunction must(name: string): string {\n const v = process.env[name]\n if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n * criteria → a JSON array of booleans (true = satisfied). Faithful to the\n * paper's combined image+script evaluation. Throws on transport failure (never\n * a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n instruction: string,\n script: string,\n criteria: string[],\n renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n const key = must('TANGLE_API_KEY')\n const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n const text =\n `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n const content: unknown[] = [{ type: 'text', text }]\n for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n const res = await fetch(`${base}/chat/completions`, {\n method: 'POST',\n headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },\n body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),\n })\n if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)\n const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n const raw = data.choices?.[0]?.message?.content ?? ''\n const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n let arr: unknown\n try {\n arr = JSON.parse(m[0].toLowerCase())\n } catch {\n return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n }\n const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n // Pad/truncate to criteria length (a short array scores the missing as fail).\n const passed = criteria.map((_, i) => bools[i] === true)\n return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n async function load(): Promise<typeof cache & object> {\n if (cache) return cache\n const path = process.env.CADBENCH_PATH\n if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n const text = await readFile(path, 'utf8')\n cache = text\n .split('\\n')\n .filter((l) => l.trim())\n .map((l) => {\n const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n })\n return cache\n }\n\n return {\n name: 'cadbench',\n\n async preflight() {\n const { execFile } = await import('node:child_process')\n const { promisify } = await import('node:util')\n const exec = promisify(execFile)\n try {\n await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n } catch (err) {\n throw new Error(\n `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n )\n }\n await load()\n },\n\n async loadTasks(opts: LoadOptions = {}) {\n let rows = await load()\n if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n // TYPE filter (Simulative|Wild) via env, applied before limit.\n const t = process.env.CADBENCH_TYPE\n if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n if (opts.limit != null) rows = rows.slice(0, opts.limit)\n return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n },\n\n async goldArtifact() {\n return undefined // no reference bpy script ships with the benchmark\n },\n\n async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n const meta = task.metadata as unknown as CadBenchMeta\n const criteria = meta.criteria ?? []\n if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n const r = await renderBpy(artifact, { views: 4 })\n if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n const score = passed.filter(Boolean).length / criteria.length\n return { resolved: score === 1, score, detail: note }\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;AC1FA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CACjF,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,oBAAoB;EAClD,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU;EAAM;EAC9E,MAAM,KAAK,UAAU;GAAE;GAAO,YAAY;GAAM,aAAa;GAAG,UAAU,CAAC;IAAE,MAAM;IAAQ;GAAQ,CAAC;EAAE,CAAC;CACzG,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAEhG,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,UAAU,EAAE,EAAE,SAAS,WAAW;CACnD,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cadgenbench-DRhczfsG.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n try {\n const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n return { code: 0, stdout, stderr }\n } catch (err) {\n const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n }\n}\n\nfunction extractPy(text: string): string {\n const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n if (_cheat != null) return _cheat\n const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n return _cheat\n}\n\nexport interface Build123dConfig {\n routerBaseUrl: string\n routerKey: string\n model: string\n rounds?: number\n /** The build123d authoring directive — the GEPA-optimizable surface. */\n directive?: string\n}\n\nexport interface Build123dShot {\n /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n artifact: string\n /** The Python source the agent wrote. */\n source: string\n trace: Span[]\n usage: { input: number; output: number }\n ok: boolean\n built: boolean\n detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n * read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n const rounds = Math.max(1, cfg.rounds ?? 2)\n const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n const trace: Span[] = []\n const runId = `cadgenbench-${task.id}`\n let ts = Date.now()\n const tick = () => (ts += 1)\n const usage = { input: 0, output: 0 }\n // Carried across rounds in closures (the round Artifact is the Python source; the\n // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n let step = ''\n let built = false\n let lastErr = ''\n\n trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n // produced) is the early-stop, modeled as a judge so default-decide stops the\n // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n const res = await runRefineLoop<string, string>({\n rounds,\n setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n prompt: (round, history) =>\n round === 1\n ? task.prompt\n : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n runShot: async (user, round, dir) => {\n const scriptPath = join(dir, 'build.py')\n const stepPath = join(dir, 'output.step')\n const { content, usage: u } = await routerChatWithUsage(cfg, [\n { role: 'system', content: sys },\n { role: 'user', content: user },\n ])\n if (u) {\n usage.input += u.input\n usage.output += u.output\n }\n const source = extractPy(content)\n trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n await writeFile(scriptPath, source)\n const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n built = got.includes('ISO-10303-21') && got.length > 200\n lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n if (built) step = got\n trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n return { artifact: source }\n },\n judge: async () => ({ valid: built }),\n teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n })\n\n return {\n artifact: step,\n source: res.final.artifact,\n trace,\n usage,\n ok: res.final.artifact.trim().length > 0,\n built,\n detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n * Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n * evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n rd, gd = Path(rd), Path(gd)\n (rd / 'result.json').write_text('{}')\n shutil.copy(gt, gd / 'ground_truth.step')\n try:\n evaluate_result(rd, gd, candidate_step=cand)\n d = json.loads((rd / 'result.json').read_text())\n print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n except Exception as e:\n print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n gtStep: string\n resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n * exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n const g = join(CGB_DIR, 'tests/fixtures/geometry')\n return [\n { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n return {\n name: 'cadgenbench',\n\n async preflight() {\n const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n )\n if (!/ok/.test(r.stdout)) {\n throw new Error(\n `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n )\n }\n },\n\n async loadTasks(opts: LoadOptions = {}) {\n // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n // trivial fixture primitives with hard multi-feature parts (real headroom).\n let tasks = fixtureTasks()\n const hard = process.env.CGB_HARD_DIR\n if (hard) {\n const { readFile } = await import('node:fs/promises')\n tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n }\n if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n },\n\n async goldArtifact() {\n return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n },\n\n async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n const cand = join(dir, 'candidate.step')\n const scorer = join(dir, 'score.py')\n try {\n await writeFile(cand, artifact)\n await writeFile(scorer, SCORE_PY)\n // xvfb: the scorer's alignment step renders; needs a display.\n const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n )\n const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n } finally {\n await rm(dir, { recursive: true, force: true }).catch(() => {})\n }\n },\n }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}
|