@tangle-network/agent-bench 0.7.1 → 0.8.0

package/src/gate.ts

@@ -32,28 +32,26 @@ import type {
   AgentSpec,
   Budget,
   CombinatorShape,
-  DefaultVerdict,
   EqualKArm,
   EqualKVerdict,
   ExecutorContext,
   ExecutorRegistry,
   Executor,
   ExecutorFactory,
-  ExecutorResult,
   Outcome,
   Persona,
-  Runtime,
   Spend,
   SupervisedResult,
   TrajectoryReport,
 } from '@tangle-network/agent-runtime/kernel'
 import {
   definePersona,
-  routerChatWithUsage,
+  createExecutor,
   equalKOnCost,
   fanout,
   InMemoryResultBlobStore,
   InMemorySpawnJournal,
+  mapExecutorResult,
   runPersonified,
   trajectoryReport,
 } from '@tangle-network/agent-runtime/kernel'
@@ -73,10 +71,6 @@ export interface BenchSolverOptions {
   readonly adapter: BenchmarkAdapter
   readonly routerBaseUrl: string
   readonly routerKey: string
-  readonly model: string
-  /** Sampling temperature. >0 is required for the blind arm to be more than k identical samples
-   *  (k copies at temperature 0 collapse to one answer — no compute control). Default 0.7. */
-  readonly temperature?: number
 }
 
 const fnv = (prefix: string, value: unknown): string => {
@@ -114,58 +108,27 @@ function extractArtifact(adapter: BenchmarkAdapter, content: string): string {
  * a judge throw rejects the leaf (the scope types it into a `down` settlement — never a silent 0).
  */
 export function benchSolveLeaf(opts: BenchSolverOptions, spec: AgentSpec, ctx: ExecutorContext): Executor<unknown> {
-  const controller = new AbortController()
-  const abortIfSignalled = () => {
-    if (ctx.signal.aborted) controller.abort()
-  }
-  abortIfSignalled()
-  if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true })
-
-  let artifact: ExecutorResult<unknown> | undefined
-
-  return {
-    runtime: 'bench-router' as Runtime,
-    async execute(task, signal): Promise<ExecutorResult<unknown>> {
+  const inner = createExecutor({
+    backend: 'router',
+    routerBaseUrl: opts.routerBaseUrl,
+    routerKey: opts.routerKey,
+  })(spec, ctx)
+  return mapExecutorResult(inner, async (result, task) => {
       const t = task as SolveTask
-      const system = spec.profile.prompt?.systemPrompt
-      const messages = [
-        ...(typeof system === 'string' && system.length > 0
-          ? [{ role: 'system', content: system }]
-          : []),
-        { role: 'user', content: t.prompt },
-      ]
-      const started = Date.now()
-      const linked = linkSignals(signal, controller.signal)
-      const chat = await routerChatWithUsage(
-        { routerBaseUrl: opts.routerBaseUrl, routerKey: opts.routerKey, model: opts.model },
-        messages,
-        { temperature: opts.temperature ?? 0.7, ...(linked ? { signal: linked } : {}) },
-      )
-      const candidate = extractArtifact(opts.adapter, chat.content)
+      const raw = result.out as { content?: unknown }
+      const content = typeof raw?.content === 'string' ? raw.content : ''
+      const candidate = extractArtifact(opts.adapter, content)
       const score = await opts.adapter.judge(t.instance, candidate)
-      const verdict: DefaultVerdict = {
-        valid: score.resolved,
-        score: score.score,
-        ...(score.detail ? { notes: score.detail } : {}),
-      }
-      const spent: Spend = {
-        iterations: 1,
-        tokens: chat.usage ? { input: chat.usage.input, output: chat.usage.output } : { input: 0, output: 0 },
-        usd: chat.costUsd ?? 0,
-        ms: Date.now() - started,
+      return {
+        outRef: fnv('bench', { id: t.instance.id, candidate }),
+        out: candidate,
+        verdict: {
+          valid: score.resolved,
+          score: score.score,
+          ...(score.detail ? { notes: score.detail } : {}),
+        },
       }
-      artifact = { outRef: fnv('bench', { id: t.instance.id, candidate }), out: candidate, verdict, spent }
-      return artifact
-    },
-    teardown(): Promise<{ destroyed: boolean }> {
-      controller.abort()
-      return Promise.resolve({ destroyed: true })
-    },
-    resultArtifact() {
-      if (!artifact) throw new Error('benchSolveLeaf: resultArtifact() read before execute()')
-      return artifact
-    },
-  }
+    })
 }
 
 /**
@@ -189,10 +152,9 @@ export function benchSolverRegistry(opts: BenchSolverOptions): ExecutorRegistry
   }
 }
 
-/** Build the solver `Persona` from the developer's `AgentProfile` + a solve-and-grade registry.
- *  The deliverable type is the candidate text (`string`); `harness: null` is nominal — the
- *  supplied registry overrides resolution, so the root never falls through to the router/sandbox
- *  built-ins. */
+/** Build the solver `Persona` from the developer's exact `AgentProfile` + a solve-and-grade
+ *  registry. The profile still declares its execution identity; `AgentSpec.harness: null` only
+ *  says that the supplied registry owns placement instead of a built-in executor. */
 export function defineSolverPersona(
   profile: AgentProfile,
   registry: ExecutorRegistry,
@@ -233,8 +195,6 @@ export interface RunGateOptions {
   readonly strategies: ReadonlyArray<string>
   readonly routerBaseUrl: string
   readonly routerKey: string
-  readonly model: string
-  readonly temperature?: number
   /** How many benchmark instances to run (the paired n). */
   readonly n?: number
   readonly ids?: string[]
@@ -447,15 +407,3 @@ export async function runGate(opts: RunGateOptions): Promise<GateReport> {
 }
 
 /** Link two abort signals into one that fires when either does; `undefined` when neither is set. */
-function linkSignals(a: AbortSignal, b: AbortSignal): AbortSignal | undefined {
-  if (a.aborted || b.aborted) {
-    const c = new AbortController()
-    c.abort()
-    return c.signal
-  }
-  const c = new AbortController()
-  const onAbort = () => c.abort()
-  a.addEventListener('abort', onAbort, { once: true })
-  b.addEventListener('abort', onAbort, { once: true })
-  return c.signal
-}

package/src/generate-eval/certify.ts

@@ -24,8 +24,8 @@ import { execSync } from 'node:child_process'
 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { dirname, join, resolve } from 'node:path'
-import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
 import { scoreTask, taskToPrompt } from '../search-bench/tasks'
+import { runBenchRouterTurn } from '../router-turn'
 import { type EvalCertification, type GeneratedEval, generatedEvalSchemaVersion, parseCandidate } from './schema'
 
 const certifierId = 'agent-runtime/generate-eval@1'
@@ -121,10 +121,18 @@ export async function discriminationGate(
   const apiKey = opts.gateApiKey ?? process.env.EVAL_GATE_API_KEY ?? process.env.TANGLE_API_KEY
   const model = opts.gateModel ?? process.env.EVAL_GATE_MODEL ?? 'deepseek-v4-flash'
   if (!apiKey) throw new Error('discrimination gate needs EVAL_GATE_API_KEY (or TANGLE_API_KEY)')
-  const res = await routerChatWithUsage({ routerBaseUrl: baseUrl, routerKey: apiKey, model }, [
-    { role: 'user', content: taskToPrompt(candidate) },
-  ])
-  const { score, reasons } = scoreTask(candidate, res.content)
+  const res = await runBenchRouterTurn(
+    {
+      routerBaseUrl: baseUrl,
+      routerKey: apiKey,
+      profile: {
+        name: 'generated-eval-parametric-check',
+        model: { provider: 'tangle-router', default: model },
+      },
+    },
+    taskToPrompt(candidate),
+  )
+  const { score, reasons } = scoreTask(candidate, res.finalText)
   return score === 0
     ? { passed: true, detail: `parametric ${model} failed as required (${reasons.join('; ')})` }
     : { passed: false, detail: `parametric ${model} SOLVED the task from memory — not search-discriminating` }

package/src/hev-eval.mts

@@ -9,19 +9,41 @@
  */
 import { readFileSync } from 'node:fs'
 import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+import { runBenchRouterTurn } from './router-turn'
 
 const SEED_INSTRUCTION =
   'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
 
-async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<string> {
-  const res = await fetch(`${base}/chat/completions`, {
-    method: 'POST',
-    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
-  })
-  if (!res.ok) return ''
-  const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  return d.choices?.[0]?.message?.content ?? ''
+async function complete(
+  base: string,
+  key: string,
+  model: string,
+  instruction: string,
+  prompt: string,
+  maxTokens: number,
+): Promise<string> {
+  try {
+    const turn = await runBenchRouterTurn(
+      {
+        routerBaseUrl: base,
+        routerKey: key,
+        profile: {
+          name: 'humaneval-worker',
+          harness: 'cli-base',
+          model: {
+            provider: 'tangle-router',
+            default: model,
+            metadata: { temperature: 0.2, maxTokens },
+          },
+          prompt: { systemPrompt: instruction },
+        },
+      },
+      prompt,
+    )
+    return turn.finalText
+  } catch {
+    return ''
+  }
 }
 
 async function main(): Promise<void> {
@@ -55,7 +77,14 @@ async function main(): Promise<void> {
       const t = tasks[i]
       i += 1
       if (!t) continue
-      const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const reply = await complete(
+        base,
+        apiKey,
+        model,
+        instruction,
+        `\`\`\`python\n${t.prompt}\`\`\``,
+        maxTokens,
+      )
       const { pass: p } = await runChecker(t, extractCode(reply))
       if (p === 1) pass += 1
       else fails.push(t.taskId)

package/src/hev-improve.mts

@@ -19,6 +19,7 @@ import {
 import {
   canonicalCandidateDigest,
   type AgentProfile,
+  agentProfileSchema,
 } from '@tangle-network/agent-interface'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
 import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
@@ -27,6 +28,7 @@ import {
   officialOptimizerModel,
   requiredTokenPricing,
 } from './official-optimizer-config.mjs'
+import { runBenchRouterTurn, withBenchProfile } from './router-turn'
 
 // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
 // solveInstruction so the baseline arm reproduces the plain-prompt denominator.
@@ -39,21 +41,26 @@ interface Completion {
   tokOut: number
 }
 
-async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<Completion> {
-  const res = await fetch(`${base}/chat/completions`, {
-    method: 'POST',
-    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
-  })
-  if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`)
-  const d = (await res.json()) as {
-    choices?: Array<{ message?: { content?: string } }>
-    usage?: { prompt_tokens?: number; completion_tokens?: number }
+async function complete(
+  base: string,
+  key: string,
+  profile: AgentProfile,
+  prompt: string,
+  maxTokens: number,
+): Promise<Completion> {
+  const result = await runBenchRouterTurn(
+    {
+      routerBaseUrl: base,
+      routerKey: key,
+      profile: withBenchProfile(profile, { temperature: 0.2, maxTokens }),
+    },
+    prompt,
+  )
+  return {
+    text: result.finalText,
+    tokIn: result.usage.input,
+    tokOut: result.usage.output,
   }
-  const text = d.choices?.[0]?.message?.content ?? ''
-  const tokIn = d.usage?.prompt_tokens ?? 0
-  const tokOut = d.usage?.completion_tokens ?? 0
-  return { text, tokIn, tokOut }
 }
 
 async function main(): Promise<void> {
@@ -115,13 +122,18 @@ async function main(): Promise<void> {
     if (instr === undefined) throw new Error('agent: candidate profile has no system prompt')
     const t = byId.get(scenario.id)
     if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
-    const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\``
+    const prompt = `\`\`\`python\n${t.prompt}\`\`\``
+    const executionProfile: AgentProfile = agentProfileSchema.parse({
+      ...candidate,
+      name: candidate.name ?? 'humaneval-improvement-worker',
+      model: { ...candidate.model, provider: 'tangle-router', default: workerModel },
+    })
     const t0 = Date.now()
     const paid = await ctx.cost.runPaidCall({
       channel: 'agent',
       actor: 'humaneval-worker',
       model: workerModel,
-      execute: () => complete(base, key, workerModel, prompt, workerMaxTokens),
+      execute: () => complete(base, key, executionProfile, prompt, workerMaxTokens),
       receipt: (result) => {
         const usageUnknown = result.tokIn === 0 && result.tokOut === 0
         return {

package/src/hev-structural.mts

@@ -46,6 +46,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { type HumanEvalTask, extractCode, loadHumanEval } from './benchmarks/humaneval'
 import { composeStrategies } from './directives'
+import { runBenchRouterTurn } from './router-turn'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
 
 const dockerImage = 'python:3.12-slim'
@@ -345,35 +346,42 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content:
   let lastErr = ''
   for (let attempt = 1; attempt <= 4; attempt += 1) {
     if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt))
-    const ctl = new AbortController()
-    const timer = setTimeout(() => ctl.abort(), 240_000)
     try {
-      const res = await fetch(`${cfg.base}/chat/completions`, {
-        method: 'POST',
-        headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' },
-        body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }),
-        signal: ctl.signal,
-      })
-      if (!res.ok) {
-        lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`
-        continue
-      }
-      const d = (await res.json()) as {
-        choices?: Array<{ message?: { content?: string } }>
-        usage?: { prompt_tokens?: number; completion_tokens?: number }
-      }
-      const content = d.choices?.[0]?.message?.content ?? ''
+      const system = messages.find((message) => message.role === 'system')?.content
+      const result = await runBenchRouterTurn(
+        {
+          routerBaseUrl: cfg.base,
+          routerKey: cfg.key,
+          profile: {
+            name: 'humaneval-structural-worker',
+            harness: 'cli-base',
+            model: {
+              provider: 'tangle-router',
+              default: cfg.model,
+              metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens },
+            },
+            ...(system ? { prompt: { systemPrompt: system } } : {}),
+          },
+          timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000),
+        },
+        { messages: messages.filter((message) => message.role !== 'system') },
+      )
+      if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
+      const content = result.finalText
       // Reasoning models starve `content` when reasoning exhausts max_tokens — an
       // empty reply is a transient fault to retry, not a candidate to score.
       if (content.trim() === '') {
         lastErr = 'empty content'
         continue
       }
-      return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 }
+      return {
+        content,
+        attempts: attempt,
+        tokensIn: result.usage.input,
+        tokensOut: result.usage.output,
+      }
     } catch (e) {
       lastErr = e instanceof Error ? e.message : String(e)
-    } finally {
-      clearTimeout(timer)
     }
   }
   throw new Error(`completion failed after retries: ${lastErr}`)

package/src/humaneval-gate.mts

@@ -15,7 +15,7 @@
  * it calls the router directly). That is the *degenerate* rollout (HARNESS.md's
  * "Terminology"): it isolates the SELECTOR with the generator unable to self-correct,
  * so it measures the selector's value at its MAXIMUM. A real rollout (an `AgentProfile`
- * through `runLoop`, `maxTurns>0` over a persistent workspace) self-verifies by
+ * through `runAgentRounds`, `maxTurns>0` over a persistent workspace) self-verifies by
  * iterating, which shrinks the external selector's job — that is the next experiment,
  * not this one. A positive result here is the science (the selector works in a
  * deployable-checker regime), not the product.
@@ -43,7 +43,12 @@
 
 import { composeStrategies } from './directives'
 import { basePrompt, type CheckResult, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
-import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
+import {
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 import { selfConsistencySelect, verifierGroundedSelect } from './selector'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
 
@@ -84,12 +89,18 @@ async function main(): Promise<void> {
   if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
   if (!Number.isInteger(offset) || offset < 0) throw new Error(`OFFSET must be a non-negative integer, got ${process.env.OFFSET}`)
 
-  const cfg: RouterConfig = { routerBaseUrl, routerKey, model }
+  const cfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('humaneval-gate-worker', model, {
+      temperature: Number(process.env.TEMPERATURE ?? '0.8'),
+    }),
+  }
 
   console.log(`=== HumanEval deployable-verifier gate · N=${n} K=${k} offset=${offset} model=${model} ===`)
   console.log(`  router=${routerBaseUrl}  docker=${dockerImage} (--network=none, timeout ${dockerTimeoutMs}ms)`)
   console.log(
-    '  regime: STATELESS single completions (maxTurns=0, no AgentProfile/sandbox) — the selector no-self-correction LOWER BOUND, not a rollout/product number',
+    '  regime: STATELESS single completions (one exact AgentProfile turn, no sandbox) — the selector no-self-correction LOWER BOUND, not a rollout/product number',
   )
 
   const tasks = await loadHumanEval(n, offset)
@@ -110,10 +121,15 @@ async function main(): Promise<void> {
   console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`)
 
   const codes = await pool(units, solveConcurrency, async (u) => {
-    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: u.prompt }], {
-      temperature: Number(process.env.TEMPERATURE ?? '0.8'),
-    })
-    return extractCode(typeof res.content === 'string' ? res.content : '')
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: cfg.routerBaseUrl,
+        routerKey: cfg.routerKey,
+        profile: withBenchProfile(cfg.profile, { name: 'humaneval-gate-worker' }),
+      },
+      u.prompt,
+    )
+    return extractCode(res.finalText)
   })
 
   console.log(`▶ running ${codes.length} candidates through the Docker deployable checker, conc=${dockerConcurrency}`)

package/src/humaneval-object-ablation.mts

@@ -19,7 +19,7 @@
  * container with a hard timeout — seconds per task. Paired
  * McNemar over the per-task pass/fail difference gives the significance.
  *
- * Run from cwd=bench:  env WORKER_MODEL=google/gemini-2.5-flash-lite N=60 K=3 \
+ * Run from cwd=bench:  env WORKER_MODEL=deepseek-v4-flash N=60 K=3 \
  *   REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts
  */
 import {
@@ -28,11 +28,13 @@ import {
   runPythonProgram,
   type HumanEvalTask,
 } from './benchmarks/humaneval'
+import { runBenchRouterTurn } from './router-turn'
 
 const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
 const KEY = process.env.TANGLE_API_KEY
 if (!KEY) throw new Error('TANGLE_API_KEY required')
-const MODEL = process.env.WORKER_MODEL ?? 'google/gemini-2.5-flash-lite'
+const ROUTER_KEY = KEY
+const MODEL = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
 const N = Number(process.env.N ?? 60)
 const OFFSET = Number(process.env.OFFSET ?? 0)
 const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms)
@@ -40,23 +42,59 @@ const REPS = Number(process.env.REPS ?? 2)
 const CONC = Number(process.env.CONCURRENCY ?? 6)
 const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000)
 
-interface ChatMsg { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string }
+interface ChatMsg extends Record<string, unknown> { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string }
 interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } }
 
 async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record<string, unknown> }[] }> {
-  const body: Record<string, unknown> = { model: MODEL, messages, temperature: 0.4 }
-  if (tools) { body.tools = tools; body.tool_choice = 'auto' }
   for (let attempt = 0; ; attempt++) {
-    let res: Response
     try {
-      res = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(60_000) })
-    } catch (e) { if (attempt >= 5) throw e; await sleep(800 * 2 ** attempt); continue }
-    if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(res.status)) { if (attempt >= 5) throw new Error(`router ${res.status} exhausted`); await sleep(800 * 2 ** attempt); continue }
-    if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`)
-    const j = (await res.json()) as { choices?: { message?: { content?: string; tool_calls?: { id: string; function: { name: string; arguments: string } }[] } }[] }
-    const m = j.choices?.[0]?.message
-    const toolCalls = (m?.tool_calls ?? []).map((t) => { let args: Record<string, unknown> = {}; try { args = JSON.parse(t.function.arguments) } catch { /* keep {} */ } return { id: t.id, name: t.function.name, args } })
-    return { content: m?.content ?? '', toolCalls }
+      const system = messages.find((message) => message.role === 'system')?.content
+      const result = await runBenchRouterTurn(
+        {
+          routerBaseUrl: ROUTER,
+          routerKey: ROUTER_KEY,
+          profile: {
+            name: 'humaneval-object-ablation-worker',
+            harness: 'cli-base',
+            model: {
+              provider: 'tangle-router',
+              default: MODEL,
+              metadata: {
+                temperature: 0.4,
+                ...(tools ? { toolChoice: 'auto' } : {}),
+              },
+            },
+            ...(system ? { prompt: { systemPrompt: system } } : {}),
+            ...(tools
+              ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) }
+              : {}),
+          },
+          ...(tools ? { tools } : {}),
+          timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 60_000),
+        },
+        { messages: messages.filter((message) => message.role !== 'system') },
+      )
+      const toolCalls = result.toolCalls.map((call) => {
+        if (call.id === undefined) {
+          throw new Error(`router tool call '${call.name}' omitted its required id`)
+        }
+        let args: Record<string, unknown> = {}
+        try {
+          args = JSON.parse(call.arguments) as Record<string, unknown>
+        } catch {
+          // Keep the empty argument object; the tool returns a useful error.
+        }
+        return { id: call.id, name: call.name, args }
+      })
+      return { content: result.finalText, toolCalls }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      const status = Number(/router (\d+)/.exec(message)?.[1])
+      const transient =
+        !Number.isFinite(status) || [408, 429, 500, 502, 503, 504, 520, 522, 524].includes(status)
+      if (!transient || attempt >= 5) throw error
+      await sleep(800 * 2 ** attempt)
+    }
   }
 }
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))