@tangle-network/agent-bench 0.7.1 → 0.8.1

package/src/quant-arena/quant-loop.mts

@@ -1,6 +1,6 @@
 /**
  * QUANT-ARENA campaign loop — the improvement loop embodied for trading
- * strategies. One command runs: strategy authors (Claude, profile-pinned)
+ * strategies. One command runs: strategy authors (Runtime profile-pinned)
  * propose candidate strategies (v2 `onBar` contract, driven incrementally by
  * driver.ts) -> every candidate passes a two-stage leak audit -> survivors
  * are scored on K bootstrap in-sample windows against the pinned baselines
@@ -15,7 +15,7 @@
  * are never the acceptance currency.
  *
  *   tsx src/quant-arena/quant-loop.mts --out <dir> [--candidates 2] [--seed 20260722]
- *        [--author-model sonnet] [--audit-model haiku] [--skip-llm-audit]
+ *        [--author-model glm-5.2] [--audit-model glm-5.2] [--skip-llm-audit]
  *
  * Kernel reuse (import, not copy — see src/swe-arena/):
  *  - cost accounting: the lib's durable CostLedger (createRunCostLedger) +
@@ -38,12 +38,12 @@ import { join } from 'node:path'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import process from 'node:process'
 import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign'
+import { agentProfileSchema, type AgentProfile } from '@tangle-network/agent-interface'
+import { collectAgentTurn, createExecutor, streamAgentTurn } from '@tangle-network/agent-runtime/kernel'
 import { loadCampaignCells } from '../swe-arena/cell-evidence.mts'
 import { reconcileCrashOrphansOnDisk } from '../swe-arena/ledger-orphans.mts'
 import { loadLedgerReceipts } from '../swe-arena/manifest.mts'
-import { loadAuthorProfile, type ProposerSpec } from '../swe-arena/proposer-fanout.mts'
-import { proposerShotEnv } from '../swe-arena/outer-loop.mts'
-import { run } from '../swe-arena/proc.ts'
+import { resolveAuthorProfile, type ProposerSpec } from '../swe-arena/proposer-fanout.mts'
 import { runBacktest, statsForRange, type BacktestConfig, type RangeStats } from './backtest.ts'
 import { loadInSample, type AlignedBars } from './data.ts'
 import { loadStrategyFile } from './driver.ts'
@@ -57,6 +57,9 @@ import * as equalWeight from './strategies/equal-weight/strategy.ts'
 import * as smaCrossover from './strategies/sma-crossover/strategy.ts'
 
 export const QUANT_PROFILES_DIR = fileURLToPath(new URL('./profiles', import.meta.url))
+const DEFAULT_AUTHOR_PROFILE = fileURLToPath(
+  new URL('../swe-arena/profiles/default-author.profile.json', import.meta.url),
+)
 
 // ---------------------------------------------------------------------------
 // Config.
@@ -88,11 +91,17 @@ export const PINNED_BASELINES: Record<string, GenerateSignals> = {
 /** The two demo author seats: the plain author and the quant lens. */
 export function defaultQuantProposers(): ProposerSpec[] {
   return [
-    { name: 'default-author', profile: 'default-author.profile.json', harness: 'claude-code' },
+    {
+      name: 'default-author',
+      profile: DEFAULT_AUTHOR_PROFILE,
+      harness: 'pi',
+      model: 'glm-5.2',
+    },
     {
       name: 'quant-researcher',
       profile: join(QUANT_PROFILES_DIR, 'quant-researcher.profile.json'),
-      harness: 'claude-code',
+      harness: 'pi',
+      model: 'glm-5.2',
       lens:
         'Favor ONE economically-motivated effect (trend, mean reversion, vol targeting) with few parameters. ' +
         'State the regime in which it should work and keep turnover low enough that 15bps a side cannot eat the edge.',
@@ -110,8 +119,8 @@ export function defaultConfig(outDir: string): QuantLoopConfig {
     warmupDays: 120,
     costBps: 10,
     slippageBps: 5,
-    authorModel: 'sonnet',
-    auditModel: 'haiku',
+    authorModel: 'glm-5.2',
+    auditModel: 'glm-5.2',
     skipLlmAudit: false,
     authorTimeoutMs: 480_000,
     auditTimeoutMs: 240_000,
@@ -183,10 +192,10 @@ export async function loadNotebookRows(notebookPath: string): Promise<Array<Reco
 const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
 
 // ---------------------------------------------------------------------------
-// Claude shots (author + auditor) — metered paid calls through the run ledger.
+// Exact-profile shots (author + auditor) — metered paid calls through Runtime + the run ledger.
 // ---------------------------------------------------------------------------
 
-interface ClaudeShotOutcome {
+interface ProfileShotOutcome {
   text: string
   model: string
   inputTokens: number
@@ -197,64 +206,81 @@ interface ClaudeShotOutcome {
 
 type Ledger = ReturnType<typeof createRunCostLedger>
 
-async function claudeShot(opts: {
+function withModel(profile: AgentProfile, model: string, name = profile.name): AgentProfile {
+  return agentProfileSchema.parse({
+    ...profile,
+    ...(name ? { name } : {}),
+    model: { ...profile.model, default: model },
+  })
+}
+
+function quantAuditProfile(model: string): AgentProfile {
+  return agentProfileSchema.parse({
+    name: 'quant-leak-auditor',
+    harness: 'pi',
+    model: { provider: 'tangle-router', default: model },
+    prompt: {
+      systemPrompt:
+        'Audit the supplied trading strategy for look-ahead bias and nondeterminism. Follow the requested JSON response contract exactly.',
+    },
+  })
+}
+
+async function profileShot(opts: {
   prompt: string
-  model: string
-  systemPrompt?: string
+  profile: AgentProfile
   timeoutMs: number
   cwd: string
-}): Promise<ClaudeShotOutcome> {
-  const argv = [
-    '-p',
-    '--output-format',
-    'json',
-    '--model',
-    opts.model,
-    // The shot is pure text generation: no filesystem, no shell, no web.
-    '--disallowed-tools',
-    'Bash,Edit,Write,Read,Glob,Grep,WebFetch,WebSearch,Task,NotebookEdit',
-    ...(opts.systemPrompt ? ['--append-system-prompt', opts.systemPrompt] : []),
-  ]
-  const res = await run('claude', argv, {
-    stdin: opts.prompt,
+}): Promise<ProfileShotOutcome> {
+  const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL
+  const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER
+  if (!bridgeUrl || !bridgeBearer) {
+    throw new Error(
+      'quant profile shots require CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER',
+    )
+  }
+  const factory = createExecutor({
+    backend: 'bridge',
+    bridgeUrl,
+    bridgeBearer,
     cwd: opts.cwd,
-    env: proposerShotEnv('claude-code'),
     timeoutMs: opts.timeoutMs,
   })
-  if (res.code !== 0) {
-    throw new Error(`claude shot exited ${res.code}${res.timedOut ? ' (timeout)' : ''}: ${(res.stderr || res.stdout).slice(0, 800)}`)
-  }
-  let parsed: Record<string, unknown>
-  try {
-    parsed = JSON.parse(res.stdout) as Record<string, unknown>
-  } catch {
-    throw new Error(`claude shot: unparseable --output-format json stdout: ${res.stdout.slice(0, 400)}`)
+  const turn = await collectAgentTurn(
+    streamAgentTurn(
+      { kind: 'executor', factory, profile: opts.profile },
+      opts.prompt,
+      { timeoutMs: opts.timeoutMs },
+    ),
+  )
+  if (turn.status !== 'completed') {
+    throw new Error(turn.error?.message ?? `quant profile shot ended with ${turn.status}`)
   }
-  if (parsed.is_error === true) throw new Error(`claude shot errored: ${String(parsed.result).slice(0, 800)}`)
-  const usage = (parsed.usage ?? {}) as Record<string, unknown>
-  const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
+  const cachedTokens = Number(turn.usage.promptCache?.readTokens ?? 0)
   return {
-    text: typeof parsed.result === 'string' ? parsed.result : '',
-    model: typeof parsed.model === 'string' ? parsed.model : opts.model,
-    inputTokens: num(usage.input_tokens),
-    outputTokens: num(usage.output_tokens),
-    cachedTokens: num(usage.cache_read_input_tokens),
-    costUsd: typeof parsed.total_cost_usd === 'number' ? parsed.total_cost_usd : null,
+    text: turn.finalText,
+    model: turn.usage.model ?? opts.profile.model?.default ?? 'unknown',
+    inputTokens: turn.usage.input,
+    outputTokens: turn.usage.output,
+    cachedTokens: Number.isFinite(cachedTokens) ? cachedTokens : 0,
+    costUsd: turn.usage.costUsd ?? null,
   }
 }
 
-async function meteredClaudeShot(
+async function meteredProfileShot(
   ledger: Ledger,
   meta: { phase: string; actor: string; tags: Record<string, string> },
-  opts: Parameters<typeof claudeShot>[0],
-): Promise<{ outcome: ClaudeShotOutcome; costUsd: number | null }> {
-  const paid = await ledger.runPaidCall<ClaudeShotOutcome>({
+  opts: Parameters<typeof profileShot>[0],
+): Promise<{ outcome: ProfileShotOutcome; costUsd: number | null }> {
+  const model = opts.profile.model?.default
+  if (!model) throw new Error('meteredProfileShot: profile.model.default is required')
+  const paid = await ledger.runPaidCall<ProfileShotOutcome>({
     channel: 'driver',
     phase: meta.phase,
     actor: meta.actor,
-    model: opts.model,
+    model,
     tags: meta.tags,
-    execute: () => claudeShot(opts),
+    execute: () => profileShot(opts),
     receipt: (v) => ({
       model: v.model,
       inputTokens: v.inputTokens,
@@ -379,12 +405,12 @@ async function llmLeakAudit(
   code: string,
 ): Promise<{ verdict: 'clean' | 'leak' | 'inconclusive'; evidence: string }> {
   for (let attempt = 0; attempt < 2; attempt++) {
-    const { outcome } = await meteredClaudeShot(
+    const { outcome } = await meteredProfileShot(
       ledger,
-      { phase: 'audit.leak', actor: 'leak-auditor:claude', tags: { candidateId, attempt: String(attempt) } },
+      { phase: 'audit.leak', actor: 'leak-auditor:runtime', tags: { candidateId, attempt: String(attempt) } },
       {
         prompt: AUDIT_PROMPT_HEADER + '```ts\n' + code + '\n```',
-        model: config.auditModel,
+        profile: quantAuditProfile(config.auditModel),
         timeoutMs: config.auditTimeoutMs,
         cwd: config.outDir,
       },
@@ -630,7 +656,9 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo
   const rows: CandidateRow[] = []
 
   for (const proposer of config.proposers) {
-    const profile = loadAuthorProfile(proposer)
+    const sourceProfile = resolveAuthorProfile(proposer)
+    if (!sourceProfile) throw new Error(`quant proposer ${proposer.name}: exact profile is required`)
+    const profile = withModel(sourceProfile, config.authorModel, `quant-${proposer.name}`)
     for (let shot = 0; shot < config.candidatesPerProposer; shot++) {
       nTried += 1
       const candidateId = `cand-${String(nTried).padStart(3, '0')}-${proposer.name}`
@@ -642,7 +670,7 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo
         at: new Date().toISOString(),
         candidateId,
         proposer: proposer.name,
-        authorModel: config.authorModel,
+        authorModel: profile.model?.default ?? config.authorModel,
         strategyPath: null,
         sha256: null,
         authoringCostUsd: null,
@@ -663,13 +691,12 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo
           nTried,
           ...(proposer.lens ? { lens: proposer.lens } : {}),
         })
-        const { outcome, costUsd } = await meteredClaudeShot(
+        const { outcome, costUsd } = await meteredProfileShot(
           ledger,
           { phase: 'search.proposal', actor: `proposer-shot:${proposer.name}`, tags: { candidateId } },
           {
             prompt,
-            model: config.authorModel,
-            ...(profile?.prompt?.systemPrompt ? { systemPrompt: profile.prompt.systemPrompt } : {}),
+            profile,
             timeoutMs: config.authorTimeoutMs,
             cwd: config.outDir,
           },
@@ -798,7 +825,7 @@ if (isMain) {
   if (!outDir) {
     console.error(
       'usage: tsx src/quant-arena/quant-loop.mts --out <dir> [--candidates 2] [--seed 20260722] ' +
-        '[--author-model sonnet] [--audit-model haiku] [--skip-llm-audit]  # SPENDS: author + audit shots',
+        '[--author-model glm-5.2] [--audit-model glm-5.2] [--skip-llm-audit]  # SPENDS: author + audit shots',
     )
     process.exit(2)
   }

package/src/research-shot.ts

@@ -1,16 +1,16 @@
 /**
  * One research rollout as a reusable primitive: 2-step RAG — (1) provider-pinned web
  * search via the router's proven `/v1/search?provider=<id>` + `web_fetch` of the top-K
- * result pages, (2) answer with that evidence via `routerChatWithUsage` (no tools on the
+ * result pages, (2) answer with that evidence through Runtime's profile-bound turn (no tools on the
  * answer call → `content` always present, so a search arm differs from the parametric
  * control ONLY by the evidence). Pure router HTTP (bearer `TANGLE_API_KEY`).
  *
  * Shared by the off-sandbox RAG leaderboard (`research-gate.mts`) and the router-backed
  * loop executor (`router-executor.ts`), so both score the identical retrieve→answer body
  * — the only difference is who drives the rounds (a flat best-of-k pool vs the real
- * `runLoop` kernel with analyst steering).
+ * `runAgentRounds` kernel with analyst steering).
  */
-import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
+import { runBenchRouterTurn } from './router-turn'
 
 export interface ShotCfg {
   model: string
@@ -109,15 +109,25 @@ export async function runResearchShot(prompt: string, taskId: string, attempt: n
         : 'Answer from your own knowledge. ') +
       'If you are not fully certain, still COMMIT to your single best estimate — never refuse, defer, or reply with a question.'
     const userContent = useSearch && context ? `${prompt}\n\n=== WEB SEARCH RESULTS (provider: ${cfg.search}) ===\n${context}` : prompt
-    const { content } = await routerChatWithUsage(
-      { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
-      [
-        { role: 'system', content: commit },
-        { role: 'user', content: userContent },
-      ],
-      { temperature: cfg.temperature, ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(cfg.timeoutMs) } : {}) },
+    const turn = await runBenchRouterTurn(
+      {
+        routerBaseUrl: cfg.routerBaseUrl,
+        routerKey: cfg.routerKey,
+        profile: {
+          name: 'research-shot-answerer',
+          harness: 'cli-base',
+          model: {
+            provider: 'tangle-router',
+            default: cfg.model,
+            metadata: { temperature: cfg.temperature },
+          },
+          prompt: { systemPrompt: commit },
+        },
+        ...(cfg.timeoutMs ? { timeoutMs: cfg.timeoutMs } : {}),
+      },
+      userContent,
     )
-    const answer = content.trim()
+    const answer = turn.finalText.trim()
     const ok = answer.length > 0
     return { taskId, attempt, answer, ok, searches, wallMs: Date.now() - startedAt, ...(ok ? {} : { detail: `empty answer (searches=${searches})` }) }
   } catch (err) {

package/src/resolve-client.ts

@@ -1,6 +1,6 @@
 /**
  * The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
- * (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
+ * (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runAgentRounds` drives from a single
  * selector instead of hand-rolling the branch:
  *
  *   - `router`               → OFF-BOX: a router chat-completion as the leaf executor, presented
@@ -11,10 +11,10 @@
  *   - `bridge`               → OFF-BOX: a local cli-bridge fronting a harness CLI
  *                              (opencode/kimi-code/…) as the leaf executor. Same resumable
  *                              `bridgeExecutor` the supervisor uses; harness+model ride the
- *                              bridge `model` id (`${harness}/${model}`).
+ *                              exact harness/provider/model come from the turn's AgentProfile.
  *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
  *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
- *                              this only decides off-box-vs-in-box transport for `runLoop`.
+ *                              this only decides off-box-vs-in-box transport for `runAgentRounds`.
  *
  * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
  * the mapping can't drift between callers.
@@ -33,7 +33,6 @@ export interface ResolveBenchClientOptions {
   backend: string
   routerBaseUrl: string
   routerKey: string
-  model: string
   /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop
    *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
   searchProvider?: string
@@ -47,7 +46,7 @@ export interface ResolveBenchClientOptions {
 }
 
 export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClient {
-  const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts
+  const { backend, routerBaseUrl, routerKey, searchProvider } = opts
   if (backend === 'router') {
     if (searchProvider) {
       return inlineSandboxClient(
@@ -55,13 +54,12 @@ export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClie
           backend: 'router-tools',
           routerBaseUrl,
           routerKey,
-          model,
           tools: [webSearchTool],
           executeToolCall: makeSearchExecutor({ routerBaseUrl, routerKey, provider: searchProvider }),
         }),
       )
     }
-    return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey, model }))
+    return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey }))
   }
   if (backend === 'bridge') {
     // bench's bearer fallback (`?? routerKey`) resolves first, then the shared
@@ -70,7 +68,7 @@ export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClie
     if (!bridgeBearer) throw new Error("resolveBenchClient: backend 'bridge' needs bridgeBearer or routerKey")
     return resolveSandboxClient({
       backend: 'bridge',
-      bridge: { url: opts.bridgeUrl, bearer: bridgeBearer, model, timeoutMs: opts.timeoutMs },
+      bridge: { url: opts.bridgeUrl, bearer: bridgeBearer, timeoutMs: opts.timeoutMs },
     })
   }
   return new Sandbox({

package/src/rollout-ledger/backfill-swe-arena.test.mts

@@ -2,7 +2,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { assertRolloutLine, claudeProjectSlug, toSftRows } from '@tangle-network/agent-eval/rollout'
+import {
+  assertMintedLines,
+  assertRolloutLine,
+  claudeProjectSlug,
+  toSftRows,
+} from '@tangle-network/agent-eval/rollout'
 import { backfillSweArena } from './backfill-swe-arena.mts'
 
 let dir: string
@@ -262,7 +267,7 @@ describe('backfillSweArena', () => {
 
     // The dataset is immediately trainable: reward==1 trainable-split lines
     // with transcripts → SFT rows (candidate worker + proposer here).
-    expect(toSftRows(lines)).toHaveLength(2)
+    expect(toSftRows(assertMintedLines(lines))).toHaveLength(2)
   })
 
   it('joins a respawned worker cwd once, not once per recovered entry', async () => {

package/src/router-executor.ts

@@ -7,7 +7,7 @@
  * Why off-sandbox: research is retrieval, not in-box code execution — it never
  * needed a box, and a real sandbox box reaches only the router (egress allowlist,
  * ops-board #976), so it cannot web-search natively anyway. Driving the loop with
- * this executor instead of a box gives the REAL `runLoop` kernel full `rounds` +
+ * this executor instead of a box gives the REAL `runAgentRounds` kernel full `rounds` +
  * analyst steering (the depth regime), search working, no sandbox dependency.
  *
  * This is a BYO `Executor` over `runResearchShot`; `inlineSandboxClient` supplies

package/src/router-turn.ts

@@ -0,0 +1,135 @@
+import {
+  type AgentProfile,
+  agentProfileSchema,
+  type ReasoningEffort,
+} from '@tangle-network/agent-interface'
+import {
+  collectAgentTurn,
+  createExecutor,
+  streamAgentTurn,
+  type CollectedAgentTurn,
+  type ToolSpec,
+} from '@tangle-network/agent-runtime/kernel'
+
+/** Bench-local target shorthand; Runtime still executes only the exact profile below. */
+export interface BenchRouterTarget {
+  routerBaseUrl: string
+  routerKey: string
+  profile: AgentProfile
+}
+
+export interface BenchRouterTurnConfig extends BenchRouterTarget {
+  tools?: ReadonlyArray<ToolSpec>
+  timeoutMs?: number
+  signal?: AbortSignal
+}
+
+export interface BenchProfileSettings {
+  systemPrompt?: string
+  temperature?: number
+  maxTokens?: number
+  retry?: {
+    maxAttempts?: number
+    initialBackoffMs?: number
+    maxBackoffMs?: number
+    jitter?: number
+    retryStatuses?: ReadonlyArray<number>
+    requestTimeoutMs?: number
+  }
+  maxTurns?: number
+  seed?: number
+  reasoningEffort?: ReasoningEffort
+  extraBody?: Readonly<Record<string, unknown>>
+  toolChoice?: 'auto' | 'required' | 'none'
+}
+
+/** Author an exact direct-Router profile for a benchmark. This is profile construction only;
+ * execution still accepts no model or generation fields outside the returned AgentProfile. */
+export function benchRouterProfile(
+  name: string,
+  model: string,
+  settings: BenchProfileSettings = {},
+): AgentProfile {
+  return withBenchProfile(
+    {
+      name,
+      harness: 'cli-base',
+      model: { provider: 'tangle-router', default: model },
+    },
+    settings,
+  )
+}
+
+/** Derive another exact profile while preserving all untouched canonical axes. */
+export function withBenchProfile(
+  base: AgentProfile,
+  settings: BenchProfileSettings & { name?: string },
+): AgentProfile {
+  const metadata = {
+    ...(base.model?.metadata ?? {}),
+    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),
+    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),
+    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),
+    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),
+    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),
+    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),
+    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),
+  }
+  return agentProfileSchema.parse({
+    ...base,
+    ...(settings.name ? { name: settings.name } : {}),
+    model: {
+      ...base.model,
+      ...(settings.reasoningEffort !== undefined
+        ? { reasoningEffort: settings.reasoningEffort }
+        : {}),
+      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
+    },
+    ...(settings.systemPrompt !== undefined
+      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }
+      : {}),
+  })
+}
+
+export function benchProfileModel(profile: AgentProfile): string {
+  const model = profile.model?.default
+  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {
+    throw new Error('benchmark AgentProfile.model.default must be concrete')
+  }
+  return model
+}
+
+/**
+ * The benchmark-side entry to Runtime's canonical one-turn path.
+ * It is only an ergonomic composition: Runtime still parses the exact profile,
+ * materializes the executor, records identity/usage/result events, and refuses
+ * profile axes the direct Router backend cannot carry.
+ */
+export async function runBenchRouterTurn(
+  config: BenchRouterTurnConfig,
+  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },
+): Promise<CollectedAgentTurn> {
+  if (!config.profile.model?.default) {
+    throw new Error('runBenchRouterTurn: profile.model.default is required')
+  }
+  const factory = createExecutor({
+    backend: 'router',
+    routerBaseUrl: config.routerBaseUrl,
+    routerKey: config.routerKey,
+    ...(config.tools ? { tools: config.tools } : {}),
+  })
+  const turn = await collectAgentTurn(
+    streamAgentTurn(
+      { kind: 'executor', factory, profile: config.profile },
+      input,
+      {
+        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
+        ...(config.signal ? { signal: config.signal } : {}),
+      },
+    ),
+  )
+  if (turn.status !== 'completed') {
+    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)
+  }
+  return turn
+}

package/src/run-benchmarks.ts

@@ -178,7 +178,6 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     backend: cell.backend ?? 'router',
     routerBaseUrl,
     routerKey,
-    model: cell.model,
     ...(bridgeUrl ? { bridgeUrl } : {}),
     ...(bridgeBearer ? { bridgeBearer } : {}),
     ...(sandboxBaseUrl ? { sandboxBaseUrl } : {}),
@@ -186,7 +185,13 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     ...(timeoutMs ? { timeoutMs } : {}),
   })
   const harness = cell.harness ?? (cell.profile?.metadata?.backendType as string | undefined) ?? 'opencode'
-  const profile: AgentProfile = cell.profile ?? { name: cell.label, metadata: { backendType: harness } }
+  const profile: AgentProfile =
+    cell.profile ?? {
+      name: cell.label,
+      harness: harness as AgentProfile['harness'],
+      model: { provider: 'tangle-router', default: cell.model },
+      metadata: { backendType: harness },
+    }
   // Unique per shot: the same (adapter, task) runs concurrently across cells and reps, so the box
   // name and runId must not collide.
   const uniq = Math.random().toString(36).slice(2, 8)