@tangle-network/agent-bench 0.7.1 → 0.8.0

package/src/commit0-gate.mts

@@ -53,10 +53,13 @@ import { mkdtemp, rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import {
+  collectAgentTurn,
+  createExecutor,
   type AgentRunSpec,
   type Deliverable,
   openSandboxRun,
   type SandboxRun,
+  streamAgentTurn,
 } from '@tangle-network/agent-runtime/kernel'
 import { Sandbox } from '@tangle-network/sandbox'
 import { createCommit0Adapter } from './benchmarks/commit0'
@@ -153,8 +156,17 @@ interface ShotCfg {
    *  registered model names (e.g. gpt-4.1). Override via WORKER_PROVIDER. */
   provider: string
   timeoutMs: number
-  /** local-backend: the opencode CLI binary (cli-bridge fallback when the sandbox is down). */
-  opencodeBin: string
+  /** Local Runtime bridge transport. The profile, not this transport, selects the harness/model. */
+  bridgeUrl?: string
+  bridgeBearer?: string
+}
+
+function workerProfile(cfg: ShotCfg, name: string) {
+  return {
+    name,
+    harness: 'opencode' as const,
+    model: { provider: cfg.provider, default: cfg.model },
+  }
 }
 
 /** The diff the in-box agent produces, read back off the box FS (+ any stream error). */
@@ -189,21 +201,15 @@ async function runShot(task: BenchTask, attempt: number, cfg: ShotCfg, steer?: s
   // other rollout (the powered-run crash).
   const controller = new AbortController()
   const timer = cfg.timeoutMs > 0 ? setTimeout(() => controller.abort(), cfg.timeoutMs) : undefined
-  // backend.model pins provider/model/baseUrl only; the platform writes the in-box
-  // provider config keyed to the box's own OPENCODE_MODEL_API_KEY. The inline
-  // profile + backend override is the same generic AgentRunSpec the runLoop kernel
-  // boots. Never inject an external key — the egress proxy 403s foreign credentials.
+  // The exact profile owns harness/provider/model. Sandbox overrides contain only box
+  // infrastructure; Runtime derives the backend from the profile and refuses conflicts.
   const agentRun: AgentRunSpec<string> = {
-    profile: { name: 'commit0-worker', metadata: { backendType: 'opencode' } },
+    profile: workerProfile(cfg, 'commit0-worker'),
     name: 'commit0-worker',
     taskToPrompt: () => '', // unused — the prompt is streamed directly by openSandboxRun
     sandboxOverrides: {
       name: `commit0-${task.id}-${attempt}-${randomSuffix()}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: {
-        type: 'opencode',
-        model: { provider: cfg.provider, model: cfg.model, baseUrl: cfg.routerBaseUrl },
-      },
     },
   }
   let run: SandboxRun<RolloutDeliverable> | undefined
@@ -310,15 +316,29 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
     if (co.code !== 0) {
       return { task, attempt, diff: '', ok: false, events: 0, wallMs: Date.now() - startedAt, detail: `git checkout ${meta.baseCommit} failed: ${co.out.trim().slice(-180)}` }
     }
-    // openai/* → route through the router (OPENAI_* env); anything else → opencode's
-    // OWN configured auth (kimi-for-coding / zai coding-plan subscriptions).
-    const env = cfg.model.startsWith('openai/')
-      ? { ...process.env, OPENAI_API_KEY: cfg.routerKey, OPENAI_BASE_URL: cfg.routerBaseUrl }
-      : process.env
     const prompt = steer ? steeredPrompt(localRolloutPrompt(meta), steer) : localRolloutPrompt(meta)
-    const oc = await sh(cfg.opencodeBin, ['run', prompt, '-m', cfg.model, '--dir', dir], { timeoutMs: cfg.timeoutMs, env })
-    const lines = oc.out.split('\n')
-    const events = lines.length
+    if (!cfg.bridgeUrl || !cfg.bridgeBearer) {
+      throw new Error('local rollout requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER')
+    }
+    const factory = createExecutor({
+      backend: 'bridge',
+      bridgeUrl: cfg.bridgeUrl,
+      bridgeBearer: cfg.bridgeBearer,
+      cwd: dir,
+      ...(cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {}),
+    })
+    const turn = await collectAgentTurn(
+      streamAgentTurn(
+        { kind: 'executor', factory, profile: workerProfile(cfg, `commit0-local-${attempt}`) },
+        prompt,
+        cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {},
+      ),
+    )
+    if (turn.status !== 'completed') {
+      throw new Error(turn.error?.message ?? `Runtime bridge ended with ${turn.status}`)
+    }
+    const traceEvents = turn.events.map((event) => JSON.stringify(event))
+    const events = turn.events.length
     // Read the diff straight from git, scoped to src_dir (excludes the .venv the agent made).
     const diffRes = await sh('bash', ['-c', `cd ${JSON.stringify(dir)} && git add -- ${JSON.stringify(meta.srcDir)} && git diff --cached -- ${JSON.stringify(meta.srcDir)}`], { timeoutMs: 60_000 })
     const diff = diffRes.out
@@ -329,10 +349,10 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
       diff,
       ok,
       events,
-      traceEvents: lines.slice(-TRACE_EVENTS_TAIL),
+      traceEvents: traceEvents.slice(-TRACE_EVENTS_TAIL),
       ...(steer ? { steer } : {}),
       wallMs: Date.now() - startedAt,
-      ...(ok ? {} : { detail: `no diff (opencode rc=${oc.code}): ${oc.out.trim().slice(-160)}` }),
+      ...(ok ? {} : { detail: `no diff (Runtime bridge completed): ${turn.finalText.trim().slice(-160)}` }),
     }
   } catch (err) {
     return { task, attempt, diff: '', ok: false, events: 0, ...(steer ? { steer } : {}), wallMs: Date.now() - startedAt, detail: `local rollout error: ${(err instanceof Error ? err.message : String(err)).slice(0, 180)}` }
@@ -349,7 +369,7 @@ async function main(): Promise<void> {
   const backend = process.env.COMMIT0_BACKEND === 'local' ? 'local' : 'sandbox'
   const n = Number(process.env.N ?? 8)
   const k = Number(process.env.K ?? 4)
-  const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'gpt-4.1')
+  const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'deepseek-v4-flash')
   const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
   // The arms under test. `random` = K independent blind shots (the equal-compute
   // control); `refineAudit` = blind shot 0, then trace-only-analyst-steered shots.
@@ -363,7 +383,6 @@ async function main(): Promise<void> {
   const needsRouterKey = backend === 'sandbox' || model.startsWith('openai/') || armNames.includes('refineAudit')
   const routerKey = needsRouterKey ? must('TANGLE_API_KEY') : (process.env.TANGLE_API_KEY ?? '')
   const sandboxBaseUrl = process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools'
-  const opencodeBin = process.env.OPENCODE_BIN ?? join(process.env.HOME ?? '', '.local/bin/opencode')
   // openai-compat = generic passthrough so cheap router models resolve in-box;
   // `openai` rejects non-registered model names. Override via WORKER_PROVIDER.
   const provider = process.env.WORKER_PROVIDER ?? 'openai-compat'
@@ -384,7 +403,17 @@ async function main(): Promise<void> {
 
   // Phase 1 — rollouts, concurrent. sandbox = remote box; local = cli-bridge (opencode
   // in a tmpdir, diff read from git). Both fault-isolated → a failure is a NO-DIFF, never a throw.
-  const cfg: ShotCfg = { sandboxBaseUrl, sandboxKey: routerKey, routerBaseUrl, routerKey, model, provider, timeoutMs, opencodeBin }
+  const cfg: ShotCfg = {
+    sandboxBaseUrl,
+    sandboxKey: routerKey,
+    routerBaseUrl,
+    routerKey,
+    model,
+    provider,
+    timeoutMs,
+    bridgeUrl: process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL,
+    bridgeBearer: process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER,
+  }
   const runRollout = backend === 'local' ? runShotLocal : runShot
   const analyze: AnalystFn = llmAnalyst({ routerBaseUrl, routerKey, model: analystModel })
   const logShot = (armName: string, s: Shot) =>

package/src/corpus.ts

@@ -23,7 +23,7 @@ import type { BenchRuntimeDecisionPoint, BenchRuntimeHookEvent } from './runtime
  *  verdict, the measured economics, and a bounded trace summary.
  *
  *  `costUsd`/`tokensIn`/`tokensOut`/`wallMs` are OPTIONAL on purpose: they are
- *  present only when the worker actually reported them (the `runLoop`/kernel
+ *  present only when the worker actually reported them (the `runAgentRounds`/kernel
  *  path). A worker that reports no usage (e.g. a raw opencode-stdout shot)
  *  OMITS them — it never writes a fabricated `0`. Absence means "unmeasured",
  *  which is honest and lets the canonical bridge below refuse to forge a
@@ -106,7 +106,7 @@ function summarizeAttempt<Task, Output>(iter: Iteration<Task, Output>): AttemptR
   }
 }
 
-/** Build a RunRecord from a runLoop result. `now` injected for determinism in tests. */
+/** Build a RunRecord from a runAgentRounds result. `now` injected for determinism in tests. */
 export function buildRunRecord<Task, Output>(args: {
   benchmark: string
   instanceId: string
@@ -247,7 +247,7 @@ export interface CorpusProjection {
  * (`costUsd`/`tokensIn`/`tokensOut`/`wallMs`/`output`) is reported in
  * `unmappable` rather than backfilled with phantom zeros. This is WHY the local
  * raw-stdout path (which omits economics) cannot feed the gate — only the
- * measured `runLoop`/sandbox path can, which is the correct, honest constraint.
+ * measured `runAgentRounds`/sandbox path can, which is the correct, honest constraint.
  */
 export async function benchRecordToCorpusRecords(
   rec: RunRecord,

package/src/david-attribution.mts

@@ -20,24 +20,43 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+import { benchRouterProfile, runBenchRouterTurn } from './router-turn'
 
-const KEY = process.env.TANGLE_API_KEY!
+function requiredEnv(name: string): string {
+  const value = process.env[name]
+  if (!value) throw new Error(`${name} required`)
+  return value
+}
+
+const KEY = requiredEnv('TANGLE_API_KEY')
 const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
 const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
 const N = Number(process.env.N ?? 8)
 const T = Number(process.env.T ?? 5)
 const NTASKS = Number(process.env.NTASKS ?? 60)
 const CONC = Number(process.env.CONCURRENCY ?? 6)
+const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
+const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
 
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
 async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
-  for (let a = 0; ; a++) {
-    try {
-      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
-      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
-      if (!r.ok) return ''
-      return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? ''
-    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  try {
+    const system = messages.find((message) => message.role === 'system')?.content
+    const turn = await runBenchRouterTurn(
+      {
+        routerBaseUrl: ROUTER,
+        routerKey: KEY,
+        profile: benchRouterProfile('david-attribution-worker', DAVID, {
+          ...(system ? { systemPrompt: system } : {}),
+          temperature: temp,
+          maxTokens: MAX_TOKENS,
+        }),
+        timeoutMs: LLM_TIMEOUT_MS,
+      },
+      { messages: messages.filter((message) => message.role !== 'system') },
+    )
+    return turn.finalText
+  } catch {
+    return ''
   }
 }
 const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))

package/src/david-goliath.mts

@@ -17,7 +17,7 @@
  * generator punch above its solo weight — the standing "verification is live" claim
  * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
  *
- * Run from cwd=bench:  env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \
+ * Run from cwd=bench:  env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \
  *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
  */
 import { execFile } from 'node:child_process'
@@ -25,18 +25,22 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+import { runBenchRouterTurn } from './router-turn'
 
 const KEY = process.env.TANGLE_API_KEY
 if (!KEY) throw new Error('TANGLE_API_KEY required')
+const ROUTER_KEY = KEY
 const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
 const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001'
+const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash'
 const N = Number(process.env.N ?? 8) // David candidate solutions
 const T = Number(process.env.T ?? 5) // David generated tests
 const NTASKS = Number(process.env.NTASKS ?? 164)
 const REPS = Number(process.env.REPS ?? 2)
 const CONC = Number(process.env.CONCURRENCY ?? 6)
 const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
+const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
+const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
 
 // Approx $/1M tokens (in,out) for cost accounting — the router does not price
 // every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
@@ -56,18 +60,33 @@ const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
 const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
 
 async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
-  for (let a = 0; ; a++) {
-    try {
-      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
-      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
-      if (!r.ok) return ''
-      const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } }
-      addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 })
-      return j.choices?.[0]?.message?.content ?? ''
-    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  try {
+    const system = messages.find((message) => message.role === 'system')?.content
+    const result = await runBenchRouterTurn(
+      {
+        routerBaseUrl: ROUTER,
+        routerKey: ROUTER_KEY,
+        profile: {
+          name: 'david-goliath-worker',
+          harness: 'cli-base',
+          model: {
+            provider: 'tangle-router',
+            default: model,
+            metadata: { temperature, maxTokens: MAX_TOKENS },
+          },
+          ...(system ? { prompt: { systemPrompt: system } } : {}),
+        },
+        timeoutMs: LLM_TIMEOUT,
+      },
+      { messages: messages.filter((message) => message.role !== 'system') },
+    )
+    if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
+    addU(usage, { in: result.usage.input, out: result.usage.output })
+    return result.finalText
+  } catch {
+    return ''
   }
 }
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
 const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
 async function runPy(program: string): Promise<{ ok: boolean }> {
   const d = mkdtempSync(join(tmpdir(), 'dg-'))

package/src/egress-probe.mts

@@ -1,5 +1,5 @@
 /**
- * Sandbox->router egress probe: spins a box and makes one authed chat call to
+ * Sandbox->router egress probe: spins a box and makes one model-free authenticated API read from
  * https://router.tangle.tools/v1 using the BOX-PROVISIONED key (OPENCODE_MODEL_API_KEY
  * inside the box). That is the sanctioned flow: the egress proxy validates/injects
  * credentials at the boundary and 403s foreign keys passed in from outside — a raw
@@ -19,8 +19,8 @@ try {
     const r = await box.exec(`curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${h}/ || echo FAIL`)
     console.log(h, '→', out(r))
   }
-  const chat = await box.exec(`curl -s --max-time 30 -X POST https://router.tangle.tools/v1/chat/completions -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -H 'Content-Type: application/json' -d '{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"say ok"}],"max_tokens":5}' -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`)
-  console.log('chat (box-provisioned key):', out(chat) || 'EMPTY — egress broken')
+  const models = await box.exec(`curl -s --max-time 30 https://router.tangle.tools/v1/models -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`)
+  console.log('models API (box-provisioned key):', out(models) || 'EMPTY — egress broken')
 } finally {
   await box.delete()
 }

package/src/examples/lean-proof-gate.mts

@@ -24,6 +24,7 @@ import {
   sampleThenRefine,
 } from '@tangle-network/agent-runtime/kernel'
 import { ensureLeanImage, leanCheck } from './lean-verify.js'
+import { benchRouterProfile } from '../router-turn'
 
 // Real, mathlib-free Lean 4 theorems (compile in core Lean). `header` is everything up to `:=`;
 // the prover supplies the proof term / tactic block. `reference` is a known-good proof used only
@@ -145,9 +146,15 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey,
-      model: process.env.WORKER_MODEL ?? 'gpt-4.1',
-      innerTurns: 8, // room to call lean_check and fix
-      temperature: 0.4,
+      workerProfile: benchRouterProfile(
+        'lean-proof-worker',
+        process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+        {
+          systemPrompt: 'Construct and verify Lean proofs with the available checker before answering.',
+          maxTurns: 8,
+          temperature: 0.4,
+        },
+      ),
     },
     strategies: [sample, refine, sampleThenRefine],
     budget: Number(process.env.BUDGET ?? 3),

package/src/examples/math-demo.mts

@@ -20,6 +20,7 @@ import {
   sample,
   sampleThenRefine,
 } from '@tangle-network/agent-runtime/kernel'
+import { benchRouterProfile } from '../router-turn'
 
 // GSM8K-style problems; meta.answer is the ground truth the deployable check compares to.
 const problems: Array<{ q: string; answer: number }> = [
@@ -95,9 +96,15 @@ async function main(): Promise<void> {
     worker: {
       routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
       routerKey,
-      model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-      innerTurns: 6,
-      temperature: 0.6,
+      workerProfile: benchRouterProfile(
+        'math-worker',
+        process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+        {
+          systemPrompt: 'Solve the requested math task with the available tools and verify the result.',
+          maxTurns: 6,
+          temperature: 0.6,
+        },
+      ),
     },
     strategies: [sample, refine, sampleThenRefine],
     budget: Number(process.env.BUDGET ?? 3),

package/src/examples/strategy-demo.mts

@@ -13,6 +13,7 @@
  *   TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
  */
 import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/kernel'
+import { benchRouterProfile } from '../router-turn'
 
 // ── 1. Implement an Environment (the only thing a new domain writes) ──────────────
 // A toy: the agent must drive a counter to exactly the target using the increment tool.
@@ -59,8 +60,7 @@ const counterEnv: Environment = {
 
 const task: AgenticTask = {
   id: 'counter-to-5',
-  systemPrompt: 'You operate a counter with tools.',
-  userPrompt: `Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`,
+  userPrompt: `You operate a counter with tools. Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`,
 }
 
 // ── 3. Author your OWN strategy in ~10 lines — the lego (no Supervisor ceremony) ──
@@ -97,8 +97,14 @@ async function main(): Promise<void> {
   const worker = {
     routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
     routerKey: process.env.TANGLE_API_KEY ?? '',
-    model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-    innerTurns: 6,
+    workerProfile: benchRouterProfile(
+      'strategy-demo-worker',
+      process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+      {
+        systemPrompt: 'Use the available tools to complete and verify the task.',
+        maxTurns: 6,
+      },
+    ),
   }
   if (!worker.routerKey) throw new Error('set TANGLE_API_KEY (the worker calls the router)')
 

package/src/fleet.mts

@@ -12,8 +12,13 @@
  *
  * Run it twice: the second run injects the first run's learnings into the workers.
  */
-import { createChatClient } from '@tangle-network/agent-eval'
-import { FileCorpus, observe, openSandboxRun, renderReport } from '@tangle-network/agent-runtime/kernel'
+import {
+  defaultAnalystInstruction,
+  FileCorpus,
+  observe,
+  openSandboxRun,
+  renderReport,
+} from '@tangle-network/agent-runtime/kernel'
 import { Sandbox } from '@tangle-network/sandbox'
 import { answerOutput, sandboxAgentRun, type WorkerBackendType } from './sandbox-run'
 
@@ -41,7 +46,7 @@ interface WorkerResult {
 
 async function runWorker(
   client: Sandbox,
-  cfg: { backendType: WorkerBackendType; model: string; routerBaseUrl: string },
+  cfg: { backendType: WorkerBackendType; model: string; provider: string; routerBaseUrl: string },
   id: string,
   task: string,
   priorLearnings: string,
@@ -51,7 +56,13 @@ async function runWorker(
   const controller = new AbortController()
   const timer = setTimeout(() => controller.abort(), Number(process.env.TIMEOUT_MS ?? 240_000))
   try {
-    const agentRun = sandboxAgentRun({ ...cfg, name: id })
+    const agentRun = sandboxAgentRun({
+      profile: {
+        name: id,
+        harness: cfg.backendType,
+        model: { provider: cfg.provider, default: cfg.model },
+      },
+    })
     const run = await openSandboxRun<string>(
       client,
       { agentRun, signal: controller.signal },
@@ -74,13 +85,19 @@ async function main(): Promise<void> {
   const routerKey = env('TANGLE_API_KEY')
   const cfg = {
     backendType: env('BACKEND', 'opencode') as WorkerBackendType,
-    model: env('MODEL', 'gpt-4.1'),
+    model: env('MODEL', 'deepseek-v4-flash'),
+    provider: env('WORKER_PROVIDER', 'openai-compat'),
     routerBaseUrl: env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1'),
   }
   const n = Math.min(Number(env('N', '2')), subtasks.length)
   const corpus = new FileCorpus(env('CORPUS', '/tmp/fleet-corpus.jsonl'))
-  const observerModel = env('OBSERVER_MODEL', 'gpt-4.1')
-  const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: cfg.routerBaseUrl, defaultModel: observerModel })
+  const observerModel = env('OBSERVER_MODEL', 'deepseek-v4-flash')
+  const observerProfile = {
+    name: 'fleet-observer',
+    harness: 'cli-base' as const,
+    model: { provider: 'tangle-router', default: observerModel },
+    prompt: { systemPrompt: defaultAnalystInstruction },
+  }
   const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey })
 
   // ── continuous: read what prior runs LEARNED, inject it into this run's workers
@@ -104,7 +121,12 @@ async function main(): Promise<void> {
     if (w.error) continue
     const ob = await observe(
       { task: w.task, output: w.output, trace: w.events, outcome: w.output ? 'passed' : 'unknown', runId: w.id },
-      { chat, model: observerModel, corpus, tags: [cfg.backendType, 'fleet'] },
+      {
+        profile: observerProfile,
+        executor: { backend: 'router', routerBaseUrl: cfg.routerBaseUrl, routerKey },
+        corpus,
+        tags: [cfg.backendType, 'fleet'],
+      },
     )
     totalLearned += ob.learned.length
     console.error(`  answer: ${w.output.slice(0, 120).replace(/\n/g, ' ')}`)

package/src/gate-cli.mts

@@ -46,9 +46,16 @@ async function main(): Promise<void> {
 
   const profile = {
     name: 'gate-solver',
-    model: { default: model },
+    harness: 'cli-base',
+    model: {
+      provider: 'tangle-router',
+      default: model,
+      metadata: {
+        temperature: Number(process.env.TEMPERATURE ?? 0.7),
+      },
+    },
     prompt: { systemPrompt: 'You are an expert agent. Produce the single best deliverable the task’s grader will accept.' },
-  } as unknown as AgentProfile
+  } satisfies AgentProfile
 
   const report = await runGate({
     adapter,
@@ -56,8 +63,6 @@ async function main(): Promise<void> {
     strategies,
     routerBaseUrl,
     routerKey,
-    model,
-    ...(process.env.TEMPERATURE ? { temperature: Number(process.env.TEMPERATURE) } : {}),
     n: Number(process.env.N ?? 20),
     ...(process.env.IDS ? { ids: process.env.IDS.split(',') } : {}),
     ...(process.env.SPLIT ? { split: process.env.SPLIT } : {}),

package/src/gate.test.mts

@@ -11,6 +11,7 @@
  */
 
 import assert from 'node:assert/strict'
+import type { AgentProfile } from '@tangle-network/agent-interface'
 import type {
   AgentSpec,
   DefaultVerdict,
@@ -80,7 +81,12 @@ function stubAdapter(n: number): BenchmarkAdapter {
   }
 }
 
-const profile = { name: 'stub-solver', model: { default: 'stub-model' } } as never
+const profile = {
+  name: 'stub-solver',
+  harness: 'pi',
+  model: { provider: 'test', default: 'stub-model' },
+  prompt: { systemPrompt: 'Solve the supplied benchmark task.' },
+} satisfies AgentProfile
 
 const report = await runGate({
   adapter: stubAdapter(5),
@@ -88,7 +94,6 @@ const report = await runGate({
   strategies: ['plain restate', 'use the STRONG verified approach', 'enumerate edge cases'],
   routerBaseUrl: 'http://unused',
   routerKey: 'unused',
-  model: 'stub-model',
   solverRegistry: stubRegistry,
 })