@tangle-network/agent-bench 0.7.0 → 0.8.0

package/src/benchmarks/cadbench.ts

@@ -14,6 +14,7 @@
 import { readFile } from 'node:fs/promises'
 import type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'
 import { renderBpy } from '../worker-blender'
+import { runBenchRouterTurn } from '../router-turn'
 
 interface CadBenchMeta {
   name: string
@@ -48,14 +49,26 @@ async function judgeCriteria(
     `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\n\nCRITERIA:\n${numbered}\n\nSCRIPT:\n\`\`\`python\n${script.slice(0, 6000)}\n\`\`\``
   const content: unknown[] = [{ type: 'text', text }]
   for (const url of renders) content.push({ type: 'image_url', image_url: { url } })
-  const res = await fetch(`${base}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
-    body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }),
-  })
-  if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`)
-  const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  const raw = data.choices?.[0]?.message?.content ?? ''
+  const turn = await runBenchRouterTurn(
+    {
+      routerBaseUrl: base,
+      routerKey: key,
+      profile: {
+        name: 'cadbench-vision-judge',
+        harness: 'cli-base',
+        model: {
+          provider: 'tangle-router',
+          default: model,
+          metadata: {
+            temperature: 0,
+            maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500),
+          },
+        },
+      },
+    },
+    { messages: [{ role: 'user', content }] },
+  )
+  const raw = turn.finalText
   const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw)
   if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }
   let arr: unknown

package/src/benchmarks/finresearchbench.ts

@@ -9,6 +9,7 @@
 
 import { readFile, stat } from 'node:fs/promises'
 import { join } from 'node:path'
+import { runBenchRouterTurn } from '../router-turn'
 import { benchRoot } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
@@ -180,22 +181,25 @@ function parseJudgeScore(content: string): { score: number; raw: unknown } {
 async function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {
   if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)
   const router = routerConfig()
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [
-        { role: 'system', content: meta.judgeSystemPrompt },
-        { role: 'user', content: fillTemplate(meta, response) },
-      ],
-    }),
-  })
-  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
-  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  const content = body.choices?.[0]?.message?.content
-  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
+  const turn = await runBenchRouterTurn(
+    {
+      routerBaseUrl: router.baseUrl,
+      routerKey: router.key,
+      profile: {
+        name: 'finresearchbench-judge',
+        harness: 'cli-base',
+        model: {
+          provider: 'tangle-router',
+          default: router.model,
+          metadata: { temperature: 0 },
+        },
+        prompt: { systemPrompt: meta.judgeSystemPrompt },
+      },
+    },
+    fillTemplate(meta, response),
+  )
+  const content = turn.finalText
+  if (!content) throw new Error('FinResearchBench judge returned no message content')
   const { score, raw } = parseJudgeScore(content)
   return {
     resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),

package/src/benchmarks/finsearchcomp.ts

@@ -32,6 +32,7 @@
 import { readFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { runBenchRouterTurn } from '../router-turn'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
 const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))
@@ -206,26 +207,25 @@ function parseJudgeOutput(content: string): { resolved: boolean; score: number;
 
 /** Run the record's own judge via the router. Fail loud on transport/parse errors. */
 async function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [
-        { role: 'system', content: meta.judgeSystemPrompt },
-        { role: 'user', content: fillJudgePrompt(meta, response) },
-      ],
-    }),
-  })
-  if (!res.ok) {
-    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
-  }
-  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  const content = body.choices?.[0]?.message?.content
-  if (typeof content !== 'string') {
-    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
-  }
+  const turn = await runBenchRouterTurn(
+    {
+      routerBaseUrl: router.baseUrl,
+      routerKey: router.key,
+      profile: {
+        name: 'finsearchcomp-judge',
+        harness: 'cli-base',
+        model: {
+          provider: 'tangle-router',
+          default: router.model,
+          metadata: { temperature: 0 },
+        },
+        prompt: { systemPrompt: meta.judgeSystemPrompt },
+      },
+    },
+    fillJudgePrompt(meta, response),
+  )
+  const content = turn.finalText
+  if (!content) throw new Error('FinSearchComp judge returned no message content')
   const { resolved, score, raw } = parseJudgeOutput(content)
   return {
     resolved,

package/src/benchmarks/frames.ts

@@ -25,6 +25,7 @@ import { readFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
+import { runBenchRouterTurn } from '../router-turn'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
 const execFileAsync = promisify(execFile)
@@ -322,24 +323,24 @@ async function tier2Judge(
   candidate: string,
   router: JudgeRouter,
 ): Promise<boolean> {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      seed: 0,
-      messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }],
-    }),
-  })
-  if (!res.ok) {
-    throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
-  }
-  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  const content = body.choices?.[0]?.message?.content
-  if (typeof content !== 'string') {
-    throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
-  }
+  const turn = await runBenchRouterTurn(
+    {
+      routerBaseUrl: router.baseUrl,
+      routerKey: router.key,
+      profile: {
+        name: 'frames-equivalence-judge',
+        harness: 'cli-base',
+        model: {
+          provider: 'tangle-router',
+          default: router.model,
+          metadata: { temperature: 0, seed: 0 },
+        },
+      },
+    },
+    JUDGE_PROMPT(question, gold, candidate),
+  )
+  const content = turn.finalText
+  if (!content) throw new Error('FRAMES Tier-2 judge returned no message content')
   const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/)
   const raw = (fenced ? fenced[1] : content)?.trim() ?? ''
   let parsed: { verdict?: unknown }

package/src/benchmarks/simpleqa.ts

@@ -30,6 +30,7 @@ import { readFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
+import { runBenchRouterTurn } from '../router-turn'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
 const execFileAsync = promisify(execFile)
@@ -195,23 +196,24 @@ async function gradeAnswer(
   predicted: string,
   router: GraderRouter,
 ): Promise<Grade> {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],
-    }),
-  })
-  if (!res.ok) {
-    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
-  }
-  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  const content = body.choices?.[0]?.message?.content
-  if (typeof content !== 'string') {
-    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
-  }
+  const turn = await runBenchRouterTurn(
+    {
+      routerBaseUrl: router.baseUrl,
+      routerKey: router.key,
+      profile: {
+        name: 'simpleqa-grader',
+        harness: 'cli-base',
+        model: {
+          provider: 'tangle-router',
+          default: router.model,
+          metadata: { temperature: 0 },
+        },
+      },
+    },
+    GRADER_PROMPT(question, gold, predicted),
+  )
+  const content = turn.finalText
+  if (!content) throw new Error('SimpleQA grader returned no message content')
   const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/)
   const raw = (fenced ? fenced[1] : content)?.trim() ?? ''
   let parsed: { grade?: unknown }

package/src/benchmarks/trata-hedge.ts

@@ -26,6 +26,7 @@
 
 import { readdirSync, readFileSync, statSync } from 'node:fs'
 import { join } from 'node:path'
+import { runBenchRouterTurn } from '../router-turn'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
 const DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'
@@ -191,25 +192,32 @@ function parseJsonFallback(raw: string): unknown {
 
 async function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {
   for (let i = 0; i < maxAttempts; i++) {
-    const res = await fetch(`${router.baseUrl}/chat/completions`, {
-      method: 'POST',
-      headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
-      body: JSON.stringify({
-        model: router.model,
-        temperature: 0,
-        max_tokens: 16384,
-        messages: [{ role: 'user', content: prompt }],
-      }),
-    })
-    if (!res.ok) {
-      if (i < maxAttempts - 1) continue
-      throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+    try {
+      const turn = await runBenchRouterTurn(
+        {
+          routerBaseUrl: router.baseUrl,
+          routerKey: router.key,
+          profile: {
+            name: 'trata-hedge-judge',
+            harness: 'cli-base',
+            model: {
+              provider: 'tangle-router',
+              default: router.model,
+              metadata: {
+                temperature: 0,
+                maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384),
+              },
+            },
+          },
+        },
+        prompt,
+      )
+      const content = turn.finalText
+      const parsed = parseJsonFallback(content)
+      if (parsed !== null) return parsed
+    } catch (error) {
+      if (i + 1 === maxAttempts) throw error
     }
-    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-    const content = body.choices?.[0]?.message?.content
-    if (typeof content !== 'string') continue
-    const parsed = parseJsonFallback(content)
-    if (parsed !== null) return parsed
   }
   return null
 }

package/src/browser/ui-reviewer.ts

@@ -3,7 +3,7 @@
  * browser-agent harness (./agent-adapter.ts).
  *
  * "More reviewers, the better": run a PANEL of UI reviewers over the same target
- * and union their findings — #114's runLoop ui-auditor, browser-agent-driver's
+ * and union their findings — #114's runAgentRounds ui-auditor, browser-agent-driver's
  * `bad design-audit`, a future reviewer — each a pluggable `UiReviewerAdapter`,
  * none of them owning the verdict.
  *

package/src/clbench-context-gate.mts

@@ -36,7 +36,12 @@ import { execFileSync } from 'node:child_process'
 import { existsSync, readFileSync } from 'node:fs'
 import { composeStrategies } from './directives'
 import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
-import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
+import {
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 import { selfConsistencySelect, verifierGroundedSelect } from './selector'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
 
@@ -48,7 +53,7 @@ function must(name: string): string {
   return v
 }
 
-interface ChatMessage {
+interface ChatMessage extends Readonly<Record<string, unknown>> {
   role: string
   content: string
 }
@@ -154,7 +159,7 @@ function parseJudge(reply: string, rubricCount: number): RubricVerdict {
 
 /** Grade one completion with the rubric judge. A judge API/parse failure is a real
  *  zero (the response could not be validated) — surfaced, never masked. */
-async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): Promise<RubricVerdict> {
+async function judgeRubrics(cfg: BenchRouterTarget, task: CtxTask, output: string): Promise<RubricVerdict> {
   if (!output.trim()) return { fraction: 0, allPass: false, graded: 0 }
   const rubricsText = task.rubrics.map((r, i) => `${i + 1}. ${r}`).join('\n')
   // Fault-isolate the judge: a transient router failure (after retries) or an
@@ -162,8 +167,18 @@ async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): P
   // NOT throw — one bad grade would otherwise crash the whole N×K×2 run. graded=0
   // marks it as judge-failed so it's distinguishable from a real 0/N rubric pass.
   try {
-    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: judgePrompt(rubricsText, output) }], { temperature: 0 })
-    return parseJudge(typeof res.content === 'string' ? res.content : '', task.rubrics.length)
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: cfg.routerBaseUrl,
+        routerKey: cfg.routerKey,
+        profile: withBenchProfile(cfg.profile, {
+          name: 'clbench-rubric-judge',
+          temperature: 0,
+        }),
+      },
+      judgePrompt(rubricsText, output),
+    )
+    return parseJudge(res.finalText, task.rubrics.length)
   } catch {
     return { fraction: 0, allPass: false, graded: 0 }
   }
@@ -190,8 +205,18 @@ async function main(): Promise<void> {
   if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
   if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
 
-  const workerCfg: RouterConfig = { routerBaseUrl, routerKey, model }
-  const judgeCfg: RouterConfig = { routerBaseUrl, routerKey, model: judgeModel }
+  const workerCfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('clbench-context-worker', model, {
+      temperature: Number(process.env.TEMPERATURE ?? '0.8'),
+    }),
+  }
+  const judgeCfg: BenchRouterTarget = {
+    routerBaseUrl,
+    routerKey,
+    profile: benchRouterProfile('clbench-rubric-judge', judgeModel, { temperature: 0 }),
+  }
 
   console.log(`=== CL-bench (Context Learning) selector gate · N=${n} K=${k} offset=${offset} ===`)
   console.log(`  worker=${model}  judge=${judgeModel} (rubric-fraction verifier)  router=${routerBaseUrl}`)
@@ -213,8 +238,19 @@ async function main(): Promise<void> {
   }
   console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`)
   const outputs = await pool(units, solveConcurrency, async (u) => {
-    const res = await routerChatWithUsage(workerCfg, u.messages, { temperature: Number(process.env.TEMPERATURE ?? '0.8') })
-    return typeof res.content === 'string' ? res.content : ''
+    const system = u.messages.find((message) => message.role === 'system')?.content
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: workerCfg.routerBaseUrl,
+        routerKey: workerCfg.routerKey,
+        profile: withBenchProfile(workerCfg.profile, {
+          name: 'clbench-context-worker',
+          ...(system ? { systemPrompt: system } : {}),
+        }),
+      },
+      { messages: u.messages.filter((message) => message.role !== 'system') },
+    )
+    return res.finalText
   })
 
   console.log(`▶ grading ${outputs.length} completions with the rubric judge (${judgeModel}), conc=${solveConcurrency}`)

package/src/cloud-loop.mts

@@ -29,8 +29,11 @@
  *   dotenvx run -f …/.env.keys -f …/agent-state.env -- \
  *     env MODEL=gpt-4.1 ROUNDS=3 pnpm exec tsx src/cloud-loop.mts
  */
-import { createChatClient } from '@tangle-network/agent-eval'
-import { observe, openSandboxRun } from '@tangle-network/agent-runtime/kernel'
+import {
+  defaultAnalystInstruction,
+  observe,
+  openSandboxRun,
+} from '@tangle-network/agent-runtime/kernel'
 import { Sandbox } from '@tangle-network/sandbox'
 import { answerOutput, sandboxAgentRun } from './sandbox-run'
 
@@ -68,11 +71,17 @@ function tools(events: ReadonlyArray<unknown>): string[] {
 
 async function main(): Promise<void> {
   const routerKey = env('TANGLE_API_KEY')
-  const model = env('MODEL', 'gpt-4.1')
+  const model = env('MODEL', 'deepseek-v4-flash')
+  const workerProvider = env('WORKER_PROVIDER', 'openai-compat')
   const routerBaseUrl = env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1')
   const rounds = Number(env('ROUNDS', '3'))
   const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey })
-  const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: routerBaseUrl, defaultModel: model })
+  const observerProfile = {
+    name: 'trace-observer',
+    harness: 'cli-base' as const,
+    model: { provider: 'tangle-router', default: model },
+    prompt: { systemPrompt: defaultAnalystInstruction },
+  }
 
   console.error(`\n=== LIVE observe→steer loop · ${model} · real cloud worker + real observer ===\n`)
   const steers: string[] = []
@@ -89,7 +98,13 @@ async function main(): Promise<void> {
     let output = ''
     let events: unknown[] = []
     try {
-      const agentRun = sandboxAgentRun({ model, routerBaseUrl, backendType: 'opencode', name: `worker-r${round}` })
+      const agentRun = sandboxAgentRun({
+        profile: {
+          name: `worker-r${round}`,
+          harness: 'opencode',
+          model: { provider: workerProvider, default: model },
+        },
+      })
       const run = await openSandboxRun<string>(
         client,
         { agentRun, signal: controller.signal },
@@ -116,7 +131,10 @@ async function main(): Promise<void> {
     // THE JOIN: a REAL observer reads the REAL trace → a finding → next round's steer.
     const ob = await observe(
       { task, output, trace: events, outcome: 'failed', runId: `r${round}` },
-      { chat, model },
+      {
+        profile: observerProfile,
+        executor: { backend: 'router', routerBaseUrl, routerKey },
+      },
     )
     const next = ob.findings.flatMap((f) => (f.recommended_action ? [f.recommended_action] : [])).slice(0, 3)
     if (next.length === 0) {

package/src/commit0-env-run.mts

@@ -12,6 +12,7 @@ import { readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { adaptiveRefine, printBenchmarkReport, refine, runBenchmark, sample, type Strategy } from '@tangle-network/agent-runtime/kernel'
 import { type Commit0Row, createCommit0Environment, rowToTask } from './commit0-env'
+import { benchRouterProfile } from './router-turn'
 
 function must(name: string): string {
   const v = process.env[name]
@@ -42,9 +43,11 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey: must('TANGLE_API_KEY'),
-      model,
-      innerTurns: Number(process.env.INNER_TURNS ?? 10),
-      temperature: 0.4,
+      workerProfile: benchRouterProfile('commit0-worker', model, {
+        systemPrompt: 'You are a senior Python engineer. Work through the available tools until the task is verified complete.',
+        maxTurns: Number(process.env.INNER_TURNS ?? 10),
+        temperature: 0.4,
+      }),
     },
     strategies,
     budget,

package/src/commit0-env.ts

@@ -65,12 +65,11 @@ function parsePytest(out: string): { passed: number; failed: number } {
 export function rowToTask(row: Commit0Row): AgenticTask {
   return {
     id: row.instance_id,
-    systemPrompt:
+    userPrompt:
       'You are a senior Python engineer implementing a stubbed library so its existing test suite passes. ' +
       'Workflow: list_files and read the tests + stubs to learn the required behavior, write COMPLETE implementations ' +
       `with write_file (source under ${row.src_dir} only — the test dir is read-only), then run_tests and fix failures. ` +
-      'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.',
-    userPrompt:
+      'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.\n\n' +
       `Implement the stubbed library "${row.repo}". The public functions/classes under \`${row.src_dir}\` have empty bodies. ` +
       `Make the existing tests under \`${row.test.test_dir}\` pass.${row.setup.specification ? ` Spec: ${row.setup.specification}` : ''}`,
     meta: { instanceId: row.instance_id },