@tangle-network/agent-bench 0.7.1 → 0.8.1

package/src/clbench-context-gate.mts

@@ -36,7 +36,12 @@ import { execFileSync } from 'node:child_process'
 import { existsSync, readFileSync } from 'node:fs'
 import { composeStrategies } from './directives'
 import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
-import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
+import {
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 import { selfConsistencySelect, verifierGroundedSelect } from './selector'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
 
@@ -48,7 +53,7 @@ function must(name: string): string {
   return v
 }
 
-interface ChatMessage {
+interface ChatMessage extends Readonly<Record<string, unknown>> {
   role: string
   content: string
 }
@@ -154,7 +159,7 @@ function parseJudge(reply: string, rubricCount: number): RubricVerdict {
 
 /** Grade one completion with the rubric judge. A judge API/parse failure is a real
  *  zero (the response could not be validated) — surfaced, never masked. */
-async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): Promise<RubricVerdict> {
+async function judgeRubrics(cfg: BenchRouterTarget, task: CtxTask, output: string): Promise<RubricVerdict> {
   if (!output.trim()) return { fraction: 0, allPass: false, graded: 0 }
   const rubricsText = task.rubrics.map((r, i) => `${i + 1}. ${r}`).join('\n')
   // Fault-isolate the judge: a transient router failure (after retries) or an
@@ -162,8 +167,18 @@ async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): P
   // NOT throw — one bad grade would otherwise crash the whole N×K×2 run. graded=0
   // marks it as judge-failed so it's distinguishable from a real 0/N rubric pass.
   try {
-    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: judgePrompt(rubricsText, output) }], { temperature: 0 })
-    return parseJudge(typeof res.content === 'string' ? res.content : '', task.rubrics.length)
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: cfg.routerBaseUrl,
+        routerKey: cfg.routerKey,
+        profile: withBenchProfile(cfg.profile, {
+          name: 'clbench-rubric-judge',
+          temperature: 0,
+        }),
+      },
+      judgePrompt(rubricsText, output),
+    )
+    return parseJudge(res.finalText, task.rubrics.length)
   } catch {
     return { fraction: 0, allPass: false, graded: 0 }
   }
@@ -190,8 +205,18 @@ async function main(): Promise<void> {
   if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
   if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
 
-  const workerCfg: RouterConfig = { routerBaseUrl, routerKey, model }
-  const judgeCfg: RouterConfig = { routerBaseUrl, routerKey, model: judgeModel }
+  const workerCfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('clbench-context-worker', model, {
+      temperature: Number(process.env.TEMPERATURE ?? '0.8'),
+    }),
+  }
+  const judgeCfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('clbench-rubric-judge', judgeModel, { temperature: 0 }),
+  }
 
   console.log(`=== CL-bench (Context Learning) selector gate · N=${n} K=${k} offset=${offset} ===`)
   console.log(`  worker=${model}  judge=${judgeModel} (rubric-fraction verifier)  router=${routerBaseUrl}`)
@@ -213,8 +238,19 @@ async function main(): Promise<void> {
   }
   console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`)
   const outputs = await pool(units, solveConcurrency, async (u) => {
-    const res = await routerChatWithUsage(workerCfg, u.messages, { temperature: Number(process.env.TEMPERATURE ?? '0.8') })
-    return typeof res.content === 'string' ? res.content : ''
+    const system = u.messages.find((message) => message.role === 'system')?.content
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: workerCfg.routerBaseUrl,
+        routerKey: workerCfg.routerKey,
+        profile: withBenchProfile(workerCfg.profile, {
+          name: 'clbench-context-worker',
+          ...(system ? { systemPrompt: system } : {}),
+        }),
+      },
+      { messages: u.messages.filter((message) => message.role !== 'system') },
+    )
+    return res.finalText
   })
 
   console.log(`▶ grading ${outputs.length} completions with the rubric judge (${judgeModel}), conc=${solveConcurrency}`)

package/src/cloud-loop.mts

@@ -29,8 +29,11 @@
  *   dotenvx run -f …/.env.keys -f …/agent-state.env -- \
  *     env MODEL=gpt-4.1 ROUNDS=3 pnpm exec tsx src/cloud-loop.mts
  */
-import { createChatClient } from '@tangle-network/agent-eval'
-import { observe, openSandboxRun } from '@tangle-network/agent-runtime/kernel'
+import {
+  defaultAnalystInstruction,
+  observe,
+  openSandboxRun,
+} from '@tangle-network/agent-runtime/kernel'
 import { Sandbox } from '@tangle-network/sandbox'
 import { answerOutput, sandboxAgentRun } from './sandbox-run'
 
@@ -68,11 +71,17 @@ function tools(events: ReadonlyArray<unknown>): string[] {
 
 async function main(): Promise<void> {
   const routerKey = env('TANGLE_API_KEY')
-  const model = env('MODEL', 'gpt-4.1')
+  const model = env('MODEL', 'deepseek-v4-flash')
+  const workerProvider = env('WORKER_PROVIDER', 'openai-compat')
   const routerBaseUrl = env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1')
   const rounds = Number(env('ROUNDS', '3'))
   const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey })
-  const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: routerBaseUrl, defaultModel: model })
+  const observerProfile = {
+    name: 'trace-observer',
+    harness: 'cli-base' as const,
+    model: { provider: 'tangle-router', default: model },
+    prompt: { systemPrompt: defaultAnalystInstruction },
+  }
 
   console.error(`\n=== LIVE observe→steer loop · ${model} · real cloud worker + real observer ===\n`)
   const steers: string[] = []
@@ -89,7 +98,13 @@ async function main(): Promise<void> {
     let output = ''
     let events: unknown[] = []
     try {
-      const agentRun = sandboxAgentRun({ model, routerBaseUrl, backendType: 'opencode', name: `worker-r${round}` })
+      const agentRun = sandboxAgentRun({
+        profile: {
+          name: `worker-r${round}`,
+          harness: 'opencode',
+          model: { provider: workerProvider, default: model },
+        },
+      })
       const run = await openSandboxRun<string>(
         client,
         { agentRun, signal: controller.signal },
@@ -116,7 +131,10 @@ async function main(): Promise<void> {
     // THE JOIN: a REAL observer reads the REAL trace → a finding → next round's steer.
     const ob = await observe(
       { task, output, trace: events, outcome: 'failed', runId: `r${round}` },
-      { chat, model },
+      {
+        profile: observerProfile,
+        executor: { backend: 'router', routerBaseUrl, routerKey },
+      },
     )
     const next = ob.findings.flatMap((f) => (f.recommended_action ? [f.recommended_action] : [])).slice(0, 3)
     if (next.length === 0) {

package/src/commit0-env-run.mts

@@ -12,6 +12,7 @@ import { readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { adaptiveRefine, printBenchmarkReport, refine, runBenchmark, sample, type Strategy } from '@tangle-network/agent-runtime/kernel'
 import { type Commit0Row, createCommit0Environment, rowToTask } from './commit0-env'
+import { benchRouterProfile } from './router-turn'
 
 function must(name: string): string {
   const v = process.env[name]
@@ -42,9 +43,11 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey: must('TANGLE_API_KEY'),
-      model,
-      innerTurns: Number(process.env.INNER_TURNS ?? 10),
-      temperature: 0.4,
+      workerProfile: benchRouterProfile('commit0-worker', model, {
+        systemPrompt: 'You are a senior Python engineer. Work through the available tools until the task is verified complete.',
+        maxTurns: Number(process.env.INNER_TURNS ?? 10),
+        temperature: 0.4,
+      }),
     },
     strategies,
     budget,

package/src/commit0-env.ts

@@ -65,12 +65,11 @@ function parsePytest(out: string): { passed: number; failed: number } {
 export function rowToTask(row: Commit0Row): AgenticTask {
   return {
     id: row.instance_id,
-    systemPrompt:
+    userPrompt:
       'You are a senior Python engineer implementing a stubbed library so its existing test suite passes. ' +
       'Workflow: list_files and read the tests + stubs to learn the required behavior, write COMPLETE implementations ' +
       `with write_file (source under ${row.src_dir} only — the test dir is read-only), then run_tests and fix failures. ` +
-      'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.',
-    userPrompt:
+      'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.\n\n' +
       `Implement the stubbed library "${row.repo}". The public functions/classes under \`${row.src_dir}\` have empty bodies. ` +
       `Make the existing tests under \`${row.test.test_dir}\` pass.${row.setup.specification ? ` Spec: ${row.setup.specification}` : ''}`,
     meta: { instanceId: row.instance_id },

package/src/commit0-gate.mts

@@ -53,10 +53,13 @@ import { mkdtemp, rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import {
+  collectAgentTurn,
+  createExecutor,
   type AgentRunSpec,
   type Deliverable,
   openSandboxRun,
   type SandboxRun,
+  streamAgentTurn,
 } from '@tangle-network/agent-runtime/kernel'
 import { Sandbox } from '@tangle-network/sandbox'
 import { createCommit0Adapter } from './benchmarks/commit0'
@@ -153,8 +156,17 @@ interface ShotCfg {
    *  registered model names (e.g. gpt-4.1). Override via WORKER_PROVIDER. */
   provider: string
   timeoutMs: number
-  /** local-backend: the opencode CLI binary (cli-bridge fallback when the sandbox is down). */
-  opencodeBin: string
+  /** Local Runtime bridge transport. The profile, not this transport, selects the harness/model. */
+  bridgeUrl?: string
+  bridgeBearer?: string
+}
+
+function workerProfile(cfg: ShotCfg, name: string) {
+  return {
+    name,
+    harness: 'opencode' as const,
+    model: { provider: cfg.provider, default: cfg.model },
+  }
 }
 
 /** The diff the in-box agent produces, read back off the box FS (+ any stream error). */
@@ -189,21 +201,15 @@ async function runShot(task: BenchTask, attempt: number, cfg: ShotCfg, steer?: s
   // other rollout (the powered-run crash).
   const controller = new AbortController()
   const timer = cfg.timeoutMs > 0 ? setTimeout(() => controller.abort(), cfg.timeoutMs) : undefined
-  // backend.model pins provider/model/baseUrl only; the platform writes the in-box
-  // provider config keyed to the box's own OPENCODE_MODEL_API_KEY. The inline
-  // profile + backend override is the same generic AgentRunSpec the runLoop kernel
-  // boots. Never inject an external key — the egress proxy 403s foreign credentials.
+  // The exact profile owns harness/provider/model. Sandbox overrides contain only box
+  // infrastructure; Runtime derives the backend from the profile and refuses conflicts.
   const agentRun: AgentRunSpec<string> = {
-    profile: { name: 'commit0-worker', metadata: { backendType: 'opencode' } },
+    profile: workerProfile(cfg, 'commit0-worker'),
     name: 'commit0-worker',
     taskToPrompt: () => '', // unused — the prompt is streamed directly by openSandboxRun
     sandboxOverrides: {
       name: `commit0-${task.id}-${attempt}-${randomSuffix()}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: {
-        type: 'opencode',
-        model: { provider: cfg.provider, model: cfg.model, baseUrl: cfg.routerBaseUrl },
-      },
     },
   }
   let run: SandboxRun<RolloutDeliverable> | undefined
@@ -310,15 +316,29 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
     if (co.code !== 0) {
       return { task, attempt, diff: '', ok: false, events: 0, wallMs: Date.now() - startedAt, detail: `git checkout ${meta.baseCommit} failed: ${co.out.trim().slice(-180)}` }
     }
-    // openai/* → route through the router (OPENAI_* env); anything else → opencode's
-    // OWN configured auth (kimi-for-coding / zai coding-plan subscriptions).
-    const env = cfg.model.startsWith('openai/')
-      ? { ...process.env, OPENAI_API_KEY: cfg.routerKey, OPENAI_BASE_URL: cfg.routerBaseUrl }
-      : process.env
     const prompt = steer ? steeredPrompt(localRolloutPrompt(meta), steer) : localRolloutPrompt(meta)
-    const oc = await sh(cfg.opencodeBin, ['run', prompt, '-m', cfg.model, '--dir', dir], { timeoutMs: cfg.timeoutMs, env })
-    const lines = oc.out.split('\n')
-    const events = lines.length
+    if (!cfg.bridgeUrl || !cfg.bridgeBearer) {
+      throw new Error('local rollout requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER')
+    }
+    const factory = createExecutor({
+      backend: 'bridge',
+      bridgeUrl: cfg.bridgeUrl,
+      bridgeBearer: cfg.bridgeBearer,
+      cwd: dir,
+      ...(cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {}),
+    })
+    const turn = await collectAgentTurn(
+      streamAgentTurn(
+        { kind: 'executor', factory, profile: workerProfile(cfg, `commit0-local-${attempt}`) },
+        prompt,
+        cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {},
+      ),
+    )
+    if (turn.status !== 'completed') {
+      throw new Error(turn.error?.message ?? `Runtime bridge ended with ${turn.status}`)
+    }
+    const traceEvents = turn.events.map((event) => JSON.stringify(event))
+    const events = turn.events.length
     // Read the diff straight from git, scoped to src_dir (excludes the .venv the agent made).
     const diffRes = await sh('bash', ['-c', `cd ${JSON.stringify(dir)} && git add -- ${JSON.stringify(meta.srcDir)} && git diff --cached -- ${JSON.stringify(meta.srcDir)}`], { timeoutMs: 60_000 })
     const diff = diffRes.out
@@ -329,10 +349,10 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
       diff,
       ok,
       events,
-      traceEvents: lines.slice(-TRACE_EVENTS_TAIL),
+      traceEvents: traceEvents.slice(-TRACE_EVENTS_TAIL),
       ...(steer ? { steer } : {}),
       wallMs: Date.now() - startedAt,
-      ...(ok ? {} : { detail: `no diff (opencode rc=${oc.code}): ${oc.out.trim().slice(-160)}` }),
+      ...(ok ? {} : { detail: `no diff (Runtime bridge completed): ${turn.finalText.trim().slice(-160)}` }),
     }
   } catch (err) {
     return { task, attempt, diff: '', ok: false, events: 0, ...(steer ? { steer } : {}), wallMs: Date.now() - startedAt, detail: `local rollout error: ${(err instanceof Error ? err.message : String(err)).slice(0, 180)}` }
@@ -349,7 +369,7 @@ async function main(): Promise<void> {
   const backend = process.env.COMMIT0_BACKEND === 'local' ? 'local' : 'sandbox'
   const n = Number(process.env.N ?? 8)
   const k = Number(process.env.K ?? 4)
-  const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'gpt-4.1')
+  const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'deepseek-v4-flash')
   const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
   // The arms under test. `random` = K independent blind shots (the equal-compute
   // control); `refineAudit` = blind shot 0, then trace-only-analyst-steered shots.
@@ -363,7 +383,6 @@ async function main(): Promise<void> {
   const needsRouterKey = backend === 'sandbox' || model.startsWith('openai/') || armNames.includes('refineAudit')
   const routerKey = needsRouterKey ? must('TANGLE_API_KEY') : (process.env.TANGLE_API_KEY ?? '')
   const sandboxBaseUrl = process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools'
-  const opencodeBin = process.env.OPENCODE_BIN ?? join(process.env.HOME ?? '', '.local/bin/opencode')
   // openai-compat = generic passthrough so cheap router models resolve in-box;
   // `openai` rejects non-registered model names. Override via WORKER_PROVIDER.
   const provider = process.env.WORKER_PROVIDER ?? 'openai-compat'
@@ -384,7 +403,17 @@ async function main(): Promise<void> {
 
   // Phase 1 — rollouts, concurrent. sandbox = remote box; local = cli-bridge (opencode
   // in a tmpdir, diff read from git). Both fault-isolated → a failure is a NO-DIFF, never a throw.
-  const cfg: ShotCfg = { sandboxBaseUrl, sandboxKey: routerKey, routerBaseUrl, routerKey, model, provider, timeoutMs, opencodeBin }
+  const cfg: ShotCfg = {
+    sandboxBaseUrl,
+    sandboxKey: routerKey,
+    routerBaseUrl,
+    routerKey,
+    model,
+    provider,
+    timeoutMs,
+    bridgeUrl: process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL,
+    bridgeBearer: process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER,
+  }
   const runRollout = backend === 'local' ? runShotLocal : runShot
   const analyze: AnalystFn = llmAnalyst({ routerBaseUrl, routerKey, model: analystModel })
   const logShot = (armName: string, s: Shot) =>

package/src/corpus.ts

@@ -23,7 +23,7 @@ import type { BenchRuntimeDecisionPoint, BenchRuntimeHookEvent } from './runtime
  *  verdict, the measured economics, and a bounded trace summary.
  *
  *  `costUsd`/`tokensIn`/`tokensOut`/`wallMs` are OPTIONAL on purpose: they are
- *  present only when the worker actually reported them (the `runLoop`/kernel
+ *  present only when the worker actually reported them (the `runAgentRounds`/kernel
  *  path). A worker that reports no usage (e.g. a raw opencode-stdout shot)
  *  OMITS them — it never writes a fabricated `0`. Absence means "unmeasured",
  *  which is honest and lets the canonical bridge below refuse to forge a
@@ -106,7 +106,7 @@ function summarizeAttempt<Task, Output>(iter: Iteration<Task, Output>): AttemptR
   }
 }
 
-/** Build a RunRecord from a runLoop result. `now` injected for determinism in tests. */
+/** Build a RunRecord from a runAgentRounds result. `now` injected for determinism in tests. */
 export function buildRunRecord<Task, Output>(args: {
   benchmark: string
   instanceId: string
@@ -247,7 +247,7 @@ export interface CorpusProjection {
  * (`costUsd`/`tokensIn`/`tokensOut`/`wallMs`/`output`) is reported in
  * `unmappable` rather than backfilled with phantom zeros. This is WHY the local
  * raw-stdout path (which omits economics) cannot feed the gate — only the
- * measured `runLoop`/sandbox path can, which is the correct, honest constraint.
+ * measured `runAgentRounds`/sandbox path can, which is the correct, honest constraint.
  */
 export async function benchRecordToCorpusRecords(
   rec: RunRecord,

package/src/david-attribution.mts

@@ -20,24 +20,43 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+import { benchRouterProfile, runBenchRouterTurn } from './router-turn'
 
-const KEY = process.env.TANGLE_API_KEY!
+function requiredEnv(name: string): string {
+  const value = process.env[name]
+  if (!value) throw new Error(`${name} required`)
+  return value
+}
+
+const KEY = requiredEnv('TANGLE_API_KEY')
 const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
 const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
 const N = Number(process.env.N ?? 8)
 const T = Number(process.env.T ?? 5)
 const NTASKS = Number(process.env.NTASKS ?? 60)
 const CONC = Number(process.env.CONCURRENCY ?? 6)
+const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
+const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
 
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
 async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
-  for (let a = 0; ; a++) {
-    try {
-      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
-      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
-      if (!r.ok) return ''
-      return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? ''
-    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  try {
+    const system = messages.find((message) => message.role === 'system')?.content
+    const turn = await runBenchRouterTurn(
+      {
+        routerBaseUrl: ROUTER,
+        routerKey: KEY,
+        profile: benchRouterProfile('david-attribution-worker', DAVID, {
+          ...(system ? { systemPrompt: system } : {}),
+          temperature: temp,
+          maxTokens: MAX_TOKENS,
+        }),
+        timeoutMs: LLM_TIMEOUT_MS,
+      },
+      { messages: messages.filter((message) => message.role !== 'system') },
+    )
+    return turn.finalText
+  } catch {
+    return ''
   }
 }
 const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))

package/src/david-goliath.mts

@@ -17,7 +17,7 @@
  * generator punch above its solo weight — the standing "verification is live" claim
  * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
  *
- * Run from cwd=bench:  env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \
+ * Run from cwd=bench:  env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \
  *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
  */
 import { execFile } from 'node:child_process'
@@ -25,18 +25,22 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+import { runBenchRouterTurn } from './router-turn'
 
 const KEY = process.env.TANGLE_API_KEY
 if (!KEY) throw new Error('TANGLE_API_KEY required')
+const ROUTER_KEY = KEY
 const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
 const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001'
+const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash'
 const N = Number(process.env.N ?? 8) // David candidate solutions
 const T = Number(process.env.T ?? 5) // David generated tests
 const NTASKS = Number(process.env.NTASKS ?? 164)
 const REPS = Number(process.env.REPS ?? 2)
 const CONC = Number(process.env.CONCURRENCY ?? 6)
 const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
+const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
+const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
 
 // Approx $/1M tokens (in,out) for cost accounting — the router does not price
 // every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
@@ -56,18 +60,33 @@ const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
 const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
 
 async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
-  for (let a = 0; ; a++) {
-    try {
-      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
-      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
-      if (!r.ok) return ''
-      const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } }
-      addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 })
-      return j.choices?.[0]?.message?.content ?? ''
-    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  try {
+    const system = messages.find((message) => message.role === 'system')?.content
+    const result = await runBenchRouterTurn(
+      {
+        routerBaseUrl: ROUTER,
+        routerKey: ROUTER_KEY,
+        profile: {
+          name: 'david-goliath-worker',
+          harness: 'cli-base',
+          model: {
+            provider: 'tangle-router',
+            default: model,
+            metadata: { temperature, maxTokens: MAX_TOKENS },
+          },
+          ...(system ? { prompt: { systemPrompt: system } } : {}),
+        },
+        timeoutMs: LLM_TIMEOUT,
+      },
+      { messages: messages.filter((message) => message.role !== 'system') },
+    )
+    if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
+    addU(usage, { in: result.usage.input, out: result.usage.output })
+    return result.finalText
+  } catch {
+    return ''
   }
 }
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
 const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
 async function runPy(program: string): Promise<{ ok: boolean }> {
   const d = mkdtempSync(join(tmpdir(), 'dg-'))

package/src/egress-probe.mts

@@ -1,5 +1,5 @@
 /**
- * Sandbox->router egress probe: spins a box and makes one authed chat call to
+ * Sandbox->router egress probe: spins a box and makes one model-free authenticated API read from
  * https://router.tangle.tools/v1 using the BOX-PROVISIONED key (OPENCODE_MODEL_API_KEY
  * inside the box). That is the sanctioned flow: the egress proxy validates/injects
  * credentials at the boundary and 403s foreign keys passed in from outside — a raw
@@ -19,8 +19,8 @@ try {
     const r = await box.exec(`curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${h}/ || echo FAIL`)
     console.log(h, '→', out(r))
   }
-  const chat = await box.exec(`curl -s --max-time 30 -X POST https://router.tangle.tools/v1/chat/completions -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -H 'Content-Type: application/json' -d '{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"say ok"}],"max_tokens":5}' -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`)
-  console.log('chat (box-provisioned key):', out(chat) || 'EMPTY — egress broken')
+  const models = await box.exec(`curl -s --max-time 30 https://router.tangle.tools/v1/models -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`)
+  console.log('models API (box-provisioned key):', out(models) || 'EMPTY — egress broken')
 } finally {
   await box.delete()
 }

package/src/examples/lean-proof-gate.mts

@@ -24,6 +24,7 @@ import {
   sampleThenRefine,
 } from '@tangle-network/agent-runtime/kernel'
 import { ensureLeanImage, leanCheck } from './lean-verify.js'
+import { benchRouterProfile } from '../router-turn'
 
 // Real, mathlib-free Lean 4 theorems (compile in core Lean). `header` is everything up to `:=`;
 // the prover supplies the proof term / tactic block. `reference` is a known-good proof used only
@@ -145,9 +146,15 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey,
-      model: process.env.WORKER_MODEL ?? 'gpt-4.1',
-      innerTurns: 8, // room to call lean_check and fix
-      temperature: 0.4,
+      workerProfile: benchRouterProfile(
+        'lean-proof-worker',
+        process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+        {
+          systemPrompt: 'Construct and verify Lean proofs with the available checker before answering.',
+          maxTurns: 8,
+          temperature: 0.4,
+        },
+      ),
     },
     strategies: [sample, refine, sampleThenRefine],
     budget: Number(process.env.BUDGET ?? 3),

package/src/examples/math-demo.mts

@@ -20,6 +20,7 @@ import {
   sample,
   sampleThenRefine,
 } from '@tangle-network/agent-runtime/kernel'
+import { benchRouterProfile } from '../router-turn'
 
 // GSM8K-style problems; meta.answer is the ground truth the deployable check compares to.
 const problems: Array<{ q: string; answer: number }> = [
@@ -95,9 +96,15 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey,
-      model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-      innerTurns: 6,
-      temperature: 0.6,
+      workerProfile: benchRouterProfile(
+        'math-worker',
+        process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+        {
+          systemPrompt: 'Solve the requested math task with the available tools and verify the result.',
+          maxTurns: 6,
+          temperature: 0.6,
+        },
+      ),
     },
     strategies: [sample, refine, sampleThenRefine],
     budget: Number(process.env.BUDGET ?? 3),

package/src/examples/strategy-demo.mts

@@ -13,6 +13,7 @@
  *   TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
  */
 import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/kernel'
+import { benchRouterProfile } from '../router-turn'
 
 // ── 1. Implement an Environment (the only thing a new domain writes) ──────────────
 // A toy: the agent must drive a counter to exactly the target using the increment tool.
@@ -59,8 +60,7 @@ const counterEnv: Environment = {
 
 const task: AgenticTask = {
   id: 'counter-to-5',
-  systemPrompt: 'You operate a counter with tools.',
-  userPrompt: `Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`,
+  userPrompt: `You operate a counter with tools. Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`,
 }
 
 // ── 3. Author your OWN strategy in ~10 lines — the lego (no Supervisor ceremony) ──
@@ -97,8 +97,14 @@ async function main(): Promise<void> {
   const worker = {
     routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
     routerKey: process.env.TANGLE_API_KEY ?? '',
-    model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-    innerTurns: 6,
+    workerProfile: benchRouterProfile(
+      'strategy-demo-worker',
+      process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+      {
+        systemPrompt: 'Use the available tools to complete and verify the task.',
+        maxTurns: 6,
+      },
+    ),
   }
   if (!worker.routerKey) throw new Error('set TANGLE_API_KEY (the worker calls the router)')