@tangle-network/agent-bench 0.7.1 → 0.8.1

package/dist/router-turn-C2wMiDoo.js

@@ -0,0 +1,31 @@
+import { collectAgentTurn, createExecutor, streamAgentTurn } from "@tangle-network/agent-runtime/kernel";
+import "@tangle-network/agent-interface";
+//#region src/router-turn.ts
+/**
+* The benchmark-side entry to Runtime's canonical one-turn path.
+* It is only an ergonomic composition: Runtime still parses the exact profile,
+* materializes the executor, records identity/usage/result events, and refuses
+* profile axes the direct Router backend cannot carry.
+*/
+async function runBenchRouterTurn(config, input) {
+	if (!config.profile.model?.default) throw new Error("runBenchRouterTurn: profile.model.default is required");
+	const turn = await collectAgentTurn(streamAgentTurn({
+		kind: "executor",
+		factory: createExecutor({
+			backend: "router",
+			routerBaseUrl: config.routerBaseUrl,
+			routerKey: config.routerKey,
+			...config.tools ? { tools: config.tools } : {}
+		}),
+		profile: config.profile
+	}, input, {
+		...config.timeoutMs === void 0 ? {} : { timeoutMs: config.timeoutMs },
+		...config.signal ? { signal: config.signal } : {}
+	}));
+	if (turn.status !== "completed") throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`);
+	return turn;
+}
+//#endregion
+export { runBenchRouterTurn as t };
+
+//# sourceMappingURL=router-turn-C2wMiDoo.js.map

package/dist/router-turn-C2wMiDoo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router-turn-C2wMiDoo.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      input,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA2GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAQzE,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY,SARR,eAAe;GAC7B,SAAS;GACT,eAAe,OAAO;GACtB,WAAW,OAAO;GAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;EAChD,CAG8B;EAAG,SAAS,OAAO;CAAQ,GACrD,OACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.7.1",
+  "version": "0.8.1",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.143.0",
-    "@tangle-network/agent-interface": "0.43.0",
-    "@tangle-network/agent-knowledge": "7.0.8",
-    "@tangle-network/sandbox": "0.17.2",
-    "@tangle-network/agent-runtime": "0.126.0"
+    "@tangle-network/agent-eval": "0.144.8",
+    "@tangle-network/agent-interface": "0.46.1",
+    "@tangle-network/agent-knowledge": "7.1.3",
+    "@tangle-network/sandbox": "0.19.4",
+    "@tangle-network/agent-runtime": "0.131.6"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",
@@ -69,7 +69,7 @@
     "run-benchmarks": "tsx src/run-benchmarks-cli.mts",
     "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
     "terminal-compare": "tsx src/terminal-compare.ts",
-    "test": "node scripts/run-package-tests.mjs && node --test scripts/wait-for-published-dependencies.test.mjs",
+    "test": "node scripts/run-package-tests.mjs && node --test scripts/run-package-tests.test.mjs scripts/wait-for-published-dependencies.test.mjs",
     "typecheck:public": "tsc -p tsconfig.public.json",
     "verify:package": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs",
     "verify:package:local-runtime": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs --local-runtime",

package/scripts/appworld_driver.py

@@ -9,10 +9,7 @@
 
 import argparse
 import json
-import os
-import re
 import sys
-import time
 
 
 def fail(msg: str) -> None:
@@ -20,169 +17,9 @@ def fail(msg: str) -> None:
     sys.exit(1)
 
 
-_CODE_RE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL)
-
-
-def _extract_code(text: str) -> str:
-    blocks = _CODE_RE.findall(text or "")
-    return (blocks[-1] if blocks else "").strip()
-
-
-def _router_chat(base: str, key: str, model: str, messages: list, timeout: float = 180.0):
-    """One router chat-completion with retry on transient/429/5xx. Returns
-    (content, input_tokens, output_tokens). Raises on exhausted retries."""
-    import httpx
-
-    url = base.rstrip("/") + "/chat/completions"
-    last = None
-    for attempt in range(4):
-        try:
-            r = httpx.post(
-                url,
-                headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
-                json={"model": model, "messages": messages},
-                timeout=timeout,
-            )
-            if r.status_code in (429, 500, 502, 503, 504):
-                last = f"{r.status_code}: {r.text[:160]}"
-                time.sleep(2**attempt)
-                continue
-            r.raise_for_status()
-            d = r.json()
-            content = (d["choices"][0]["message"].get("content") or "")
-            usage = d.get("usage") or {}
-            return content, int(usage.get("prompt_tokens", 0) or 0), int(usage.get("completion_tokens", 0) or 0)
-        except Exception as e:  # noqa: BLE001
-            last = str(e)
-            if attempt < 3:
-                time.sleep(2**attempt)
-                continue
-            raise RuntimeError(f"router_chat failed after retries: {last}")
-    raise RuntimeError(f"router_chat exhausted: {last}")
-
-
-def _build_system(directive: str, world) -> str:
-    sup = world.task.supervisor
-    apps = list(getattr(world.task, "allowed_apps", []) or [])
-    descs = getattr(world.task, "app_descriptions", "")
-    desc_str = json.dumps(descs) if isinstance(descs, (dict, list)) else str(descs)
-    return (
-        f"You are an AI agent completing a digital task for your supervisor "
-        f"{getattr(sup, 'first_name', '')} {getattr(sup, 'last_name', '')} "
-        f"(email {getattr(sup, 'email', '')}, phone {getattr(sup, 'phone_number', '')}) "
-        "by WRITING PYTHON that calls app APIs (the apis.<app>.<function>(...) surface).\n\n"
-        f"Available apps: {', '.join(apps)}.\n"
-        f"App descriptions: {desc_str[:1500]}\n\n"
-        "How to work, one step per turn:\n"
-        "- Discover APIs with apis.api_docs.show_api_descriptions(app_name='<app>') and "
-        "apis.api_docs.show_api_doc(app_name='<app>', api_name='<api>') BEFORE calling them.\n"
-        "- Get the supervisor's app passwords with apis.supervisor.show_account_passwords(), then log in "
-        "to each app you use to obtain its access_token.\n"
-        "- Write ONE short Python code block per turn. After it runs you SEE its OUTPUT (or error "
-        "traceback) — use that to decide the next step. Print intermediate values you need.\n"
-        "- Iterate: inspect -> authenticate -> act -> verify. Do not guess API names or arguments.\n"
-        "- When the task is fully done call apis.supervisor.complete_task(answer=<answer>) (include the "
-        "answer if the task asks a question, otherwise apis.supervisor.complete_task()).\n"
-        "- Reply with EXACTLY ONE fenced ```python block per turn and nothing else.\n\n"
-        f"{directive}"
-    )
-
-
-def cmd_react(args) -> None:
-    """Multi-turn REPL agent: the model writes a python block, the engine executes
-    it in the PERSISTENT world, the output is fed back, and it iterates until it
-    completes the task or hits max-turns. Then AppWorld's own evaluator scores it.
-    Config (directive, model, router creds, max_turns) arrives as JSON on stdin so
-    the candidate directive can be arbitrarily long. The directive is the optimized
-    surface; the loop + contract are fixed."""
-    cfg = {}
-    raw = sys.stdin.read()
-    if raw.strip():
-        try:
-            cfg = json.loads(raw)
-        except Exception as e:  # noqa: BLE001
-            fail(f"react config JSON parse failed: {e}")
-    directive = str(cfg.get("directive", ""))
-    model = str(cfg.get("model", "gpt-4o"))
-    max_turns = int(cfg.get("max_turns", 8))
-    router_base = str(cfg.get("router_base", "https://router.tangle.tools/v1"))
-    router_key = str(cfg.get("router_key") or os.environ.get("TANGLE_API_KEY", ""))
-    if not router_key:
-        fail("react: router_key/TANGLE_API_KEY required")
-
-    try:
-        from appworld import AppWorld
-    except Exception as e:  # noqa: BLE001
-        fail(f"appworld import failed: {e}")
-
-    in_tok = 0
-    out_tok = 0
-    turns = 0
-    turns_log: list = []
-    try:
-        with AppWorld(
-            task_id=args.task_id,
-            experiment_name="bench-react",
-            raise_on_failure=False,
-        ) as world:
-            messages = [
-                {"role": "system", "content": _build_system(directive, world)},
-                {"role": "user", "content": f"Task: {world.task.instruction}"},
-            ]
-            for turn in range(max_turns):
-                turns = turn + 1
-                content, ui, uo = _router_chat(router_base, router_key, model, messages)
-                in_tok += ui
-                out_tok += uo
-                code = _extract_code(content)
-                messages.append({"role": "assistant", "content": content})
-                if not code:
-                    messages.append({
-                        "role": "user",
-                        "content": "Reply with exactly one ```python block that makes progress, "
-                        "or call apis.supervisor.complete_task().",
-                    })
-                    continue
-                output = world.execute(code)
-                turns_log.append({"code": code[:600], "output": str(output)[:600]})
-                messages.append({"role": "user", "content": "OUTPUT:\n" + str(output)[:4000]})
-                if world.task_completed():
-                    break
-            evaluation = world.evaluate().to_dict()
-    except Exception as e:  # noqa: BLE001
-        fail(f"react of {args.task_id} failed: {e}")
-
-    if "success" not in evaluation or "num_tests" not in evaluation:
-        fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}")
-    passes = evaluation.get("passes", [])
-    failures = evaluation.get("failures", [])
-    n_pass = len(passes) if isinstance(passes, list) else int(passes or 0)
-    n_fail = len(failures) if isinstance(failures, list) else int(failures or 0)
-    print(
-        json.dumps(
-            {
-                "success": bool(evaluation["success"]),
-                "passes": n_pass,
-                "fails": n_fail,
-                "num_tests": int(evaluation["num_tests"]),
-                # Failed sub-test names — the evidence a trace analyst steers on.
-                "failure_names": [str(f)[:160] for f in failures][:8]
-                if isinstance(failures, list)
-                else [],
-                "turns": turns,
-                "input_tokens": in_tok,
-                "output_tokens": out_tok,
-                "transcript": "\n---\n".join(
-                    f"CODE:\n{t['code']}\nOUTPUT:\n{t['output']}" for t in turns_log[-3:]
-                )[:1600],
-            }
-        )
-    )
-
-
 def cmd_session(args) -> None:
     """Dumb world shim: a persistent AppWorld session driven over stdin JSONL.
-    NO LLM calls here — the agent loop lives in the runtime (routerToolLoop);
+    NO LLM calls here — the profile-bound agent loop lives in Runtime;
     this process only owns world state. One JSON object per line, both ways:
       {"op":"execute","code":"..."} -> {"output":"...","task_completed":bool}
       {"op":"evaluate"}             -> the evaluate verdict JSON (+failure_names)
@@ -336,10 +173,6 @@ def main() -> None:
     p_eval.add_argument("--task-id", required=True)
     p_eval.add_argument("--split", required=True)
 
-    p_react = sub.add_parser("react")
-    p_react.add_argument("--task-id", required=True)
-    p_react.add_argument("--split", required=True)
-
     p_session = sub.add_parser("session")
     p_session.add_argument("--task-id", required=True)
     p_session.add_argument("--split", required=True)
@@ -349,8 +182,6 @@ def main() -> None:
         cmd_load(args)
     elif args.cmd == "evaluate":
         cmd_evaluate(args)
-    elif args.cmd == "react":
-        cmd_react(args)
     elif args.cmd == "session":
         cmd_session(args)
 

package/scripts/run-package-tests.mjs

@@ -8,6 +8,19 @@ const execFileAsync = promisify(execFile)
 const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
 const sourceDir = path.join(benchDir, 'src')
 
+export function resolvePackageTestTimeoutMs(env = process.env) {
+  const raw = env.AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS
+  if (raw === undefined) return undefined
+  if (!/^(?:0|[1-9]\d*)$/.test(raw)) {
+    throw new Error('AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS must be a non-negative safe integer')
+  }
+  const timeoutMs = Number(raw)
+  if (!Number.isSafeInteger(timeoutMs)) {
+    throw new Error('AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS must be a non-negative safe integer')
+  }
+  return timeoutMs
+}
+
 async function collectTests(dir) {
   const files = []
   for (const entry of await readdir(dir, { withFileTypes: true })) {
@@ -18,13 +31,14 @@ async function collectTests(dir) {
   return files.sort()
 }
 
-async function run(command, args, env = process.env) {
+export async function run(command, args, env = process.env) {
+  const timeoutMs = resolvePackageTestTimeoutMs(env)
   try {
     await execFileAsync(command, args, {
       cwd: benchDir,
       env,
       maxBuffer: 10 * 1024 * 1024,
-      timeout: 120_000,
+      ...(timeoutMs === undefined ? {} : { timeout: timeoutMs }),
     })
   } catch (error) {
     if (error?.stdout) process.stdout.write(error.stdout)
@@ -35,44 +49,50 @@ async function run(command, args, env = process.env) {
   }
 }
 
-const python = path.join(benchDir, '.venv', 'bin', 'python')
-try {
-  await access(python)
-} catch {
-  await run('python3', ['-m', 'venv', '.venv'])
-}
+async function main() {
+  const python = path.join(benchDir, '.venv', 'bin', 'python')
+  try {
+    await access(python)
+  } catch {
+    await run('python3', ['-m', 'venv', '.venv'])
+  }
 
-const tests = await collectTests(sourceDir)
-if (tests.length === 0) throw new Error('no package tests found under src/')
+  const tests = await collectTests(sourceDir)
+  if (tests.length === 0) throw new Error('no package tests found under src/')
 
-// Two test runtimes coexist under src/: node:test files run under `node --test`;
-// vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest
-// worker), so partition by the framework each file actually imports.
-const nodeTests = []
-const vitestTests = []
-for (const file of tests) {
-  const body = await readFile(file, 'utf8')
-  if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file)
-  else nodeTests.push(file)
-}
+  // Two test runtimes coexist under src/: node:test files run under `node --test`;
+  // vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest
+  // worker), so partition by the framework each file actually imports.
+  const nodeTests = []
+  const vitestTests = []
+  for (const file of tests) {
+    const body = await readFile(file, 'utf8')
+    if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file)
+    else nodeTests.push(file)
+  }
+
+  if (nodeTests.length > 0) {
+    await run(
+      process.execPath,
+      ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
+      {
+        ...process.env,
+        TSX_TSCONFIG_PATH: 'tsconfig.public.json',
+      },
+    )
+  }
+
+  if (vitestTests.length > 0) {
+    await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
+  }
+
+  await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
 
-if (nodeTests.length > 0) {
-  await run(
-    process.execPath,
-    ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
-    {
-      ...process.env,
-      TSX_TSCONFIG_PATH: 'tsconfig.public.json',
-    },
+  console.log(
+    `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,
   )
 }
 
-if (vitestTests.length > 0) {
-  await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
+if (path.resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) {
+  await main()
 }
-
-await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
-
-console.log(
-  `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,
-)

package/scripts/run-package-tests.test.mjs

@@ -0,0 +1,33 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs'
+
+test('package test timeout is optional and caller-controlled', () => {
+  assert.equal(resolvePackageTestTimeoutMs({}), undefined)
+  assert.equal(resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '0' }), 0)
+  assert.equal(resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '900000' }), 900_000)
+
+  for (const value of ['', '-1', '1.5', 'Infinity', '9007199254740992']) {
+    assert.throws(
+      () => resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: value }),
+      /must be a non-negative safe integer/,
+    )
+  }
+})
+
+test('package test timeout reaches the child process', async () => {
+  const env = {
+    ...process.env,
+    AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '100',
+  }
+  const startedAt = Date.now()
+
+  await assert.rejects(
+    run(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], env),
+    (error) => {
+      assert.equal(error.cause?.killed, true)
+      return true
+    },
+  )
+  assert.ok(Date.now() - startedAt < 2_000)
+})

package/scripts/trata-hedge/README.md

@@ -21,8 +21,8 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac
 
 ## Status (2026-06-06): pipeline PROVEN end-to-end
 
-Our solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link
-works. The naive **single-shot** baseline (gpt-4o, ~3 of N corpus files in one context
+Our Runtime-backed solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link
+works. The naive **single-shot** baseline (DeepSeek V4 Flash, ~3 of N corpus files in one context
 window) scores **0/4** — a floor: it hit only 1/3 moves on a few themes with
 hallucinations flagged, because it could not explore the full corpus. The bench is built
 for **agentic** exploration; a fair baseline needs our sandbox runtime as the solver
@@ -35,7 +35,7 @@ for **agentic** exploration; a fair baseline needs our sandbox runtime as the so
 git clone https://github.com/Trata-Inc/trata-hedge-bench /tmp/thb
 
 dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
-  bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/<env-name> gpt-4o
+  bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/<env-name> deepseek-v4-flash
 ```
 
 ## Gotchas (each cost a debugging cycle)

package/scripts/trata-hedge/run.sh

@@ -17,14 +17,18 @@
 set -euo pipefail
 
 ENV="${1:?usage: run.sh <trata-env-dir> [model]}"
-MODEL="${2:-${WORKER_MODEL:-gpt-4o}}"
+MODEL="${2:-${WORKER_MODEL:-}}"
+if [[ -z "$MODEL" ]]; then
+  echo "WORKER_MODEL is required (or pass it as argument 2)" >&2
+  exit 2
+fi
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ANS=/tmp/thb-answer.txt
 REWARD=/tmp/thb-reward.txt
 DETAILS=/tmp/thb-details.json
 
 echo "[trata] solve: $ENV (model=$MODEL)"
-WORKER_MODEL="$MODEL" python3 "$HERE/solve.py" "$ENV" "$ANS"
+WORKER_MODEL="$MODEL" pnpm --dir "$HERE/../../.." exec tsx "$HERE/../../src/trata-hedge-solve.mts" "$ENV" "$ANS"
 
 sudo -n mkdir -p /app && sudo -n cp "$ANS" /app/answer.txt && sudo -n chmod 644 /app/answer.txt
 

package/src/adapters.ts

@@ -21,6 +21,8 @@ import { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'
 import { createFramesAdapter } from './benchmarks/frames'
 import { createHotpotqaAdapter } from './benchmarks/hotpotqa'
 import { createHumanEvalAdapter } from './benchmarks/humaneval'
+import { createMcadBenchAdapter } from './benchmarks/mcad-bench'
+import { createMcadCqAdapter } from './benchmarks/mcad-cq-bench'
 import { createMind2WebAdapter } from './benchmarks/mind2web'
 import { createNoMiraclAdapter } from './benchmarks/nomiracl'
 import { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'
@@ -59,6 +61,15 @@ export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
   'appworld-react': createAppWorldReactAdapter,
   'enterpriseops-gym': createEnterpriseOpsGymAdapter,
   'cad-design': createCadDesignAdapter,
+  // Dimensioned mechanical parts: same OpenSCAD+xvfb judge deps as cad-design, but
+  // the spec is metric (bbox / volume / body count / point-in-solid hole probes)
+  // rather than qualitative, and every task carries a verified gold.
+  mcad: createMcadBenchAdapter,
+  // The same 10 parts and the same spec assertions, but the worker writes a Python
+  // CadQuery script that must export STEP as well as the ASCII STL — which closes
+  // the format deviation `mcad-tasks.ts` documents (upstream asks for STEP;
+  // OpenSCAD cannot emit it). STEP delivery is one extra SCORED check.
+  'mcad-cq': createMcadCqAdapter,
   cadbench: createCadBenchAdapter,
   cadgenbench: createCadGenBenchAdapter,
   frames: createFramesAdapter,

package/src/aec-gate.mts

@@ -22,7 +22,13 @@ import { resolveAdapter } from './adapters'
 import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
 import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
 import { composeStrategies } from './directives'
-import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
+import {
+  benchProfileModel,
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 import { pool } from './stats.mts'
 
 function must(name: string): string {
@@ -52,7 +58,7 @@ interface AttemptOutcome {
 }
 
 async function runAttempt(
-  cfg: RouterConfig,
+  cfg: BenchRouterTarget,
   adapter: BenchmarkAdapter,
   task: BenchTask,
   prompt: string,
@@ -65,8 +71,15 @@ async function runAttempt(
   let lastErr: unknown
   for (let attempt = 0; attempt < 3; attempt += 1) {
     try {
-      const res = await routerChatWithUsage(cfg, [{ role: 'user', content: prompt }])
-      const content = typeof res.content === 'string' ? res.content : ''
+      const res = await runBenchRouterTurn(
+        {
+          routerBaseUrl: cfg.routerBaseUrl,
+          routerKey: cfg.routerKey,
+          profile: withBenchProfile(cfg.profile, { name: 'aec-worker' }),
+        },
+        prompt,
+      )
+      const content = res.finalText
       const verdict = await adapter.judge(task, content)
       return {
         prompt,
@@ -74,8 +87,10 @@ async function runAttempt(
         score: verdict.score,
         resolved: verdict.resolved,
         wallMs: Date.now() - startedAt,
-        ...(res.costUsd !== undefined ? { costUsd: res.costUsd } : {}),
-        ...(res.usage ? { tokensIn: res.usage.input, tokensOut: res.usage.output } : {}),
+        ...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}),
+        ...(res.usage.tokensKnown === false
+          ? {}
+          : { tokensIn: res.usage.input, tokensOut: res.usage.output }),
       }
     } catch (err) {
       lastErr = err
@@ -115,7 +130,7 @@ interface ArmResult {
 
 async function runArm(
   arm: ArmSpec,
-  cfg: RouterConfig,
+  cfg: BenchRouterTarget,
   adapter: BenchmarkAdapter,
   tasks: BenchTask[],
   k: number,
@@ -144,7 +159,7 @@ async function runArm(
       benchmark: adapter.name,
       instanceId: task.id,
       condition: arm.condition,
-      model: cfg.model,
+      model: benchProfileModel(cfg.profile),
       // k-attempt outcome = any usable attempt resolved (the oracle@k ceiling for
       // this run; the deployable selector is scored separately by corpus-replay).
       resolved: taskOutcomes.some((o) => o.resolved),
@@ -175,7 +190,13 @@ async function main(): Promise<void> {
   if (!Number.isFinite(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
   if (!Number.isFinite(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
 
-  const cfg: RouterConfig = { routerBaseUrl, routerKey, model }
+  const cfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('aec-worker', model, {
+      retry: { maxAttempts: Number(process.env.MAX_ATTEMPTS ?? 3) },
+    }),
+  }
   const bench = process.env.BENCH ?? 'aec-bench'
   const adapter = resolveAdapter(bench)