@tangle-network/agent-bench 0.7.1 → 0.8.0

package/src/atom-humaneval.mts

@@ -21,20 +21,23 @@ import {
   type AgentProfile,
   type AgentSpec,
   contentAddress,
-  type DriverAgentOptions,
-  driverAgent,
+  createExecutor,
   createExecutorRegistry,
   createSupervisor,
-  type Executor,
-  type ExecutorResult,
   gateOnDeliverable,
   InMemoryResultBlobStore,
   InMemorySpawnJournal,
-  type RouterConfig,
-  routerBrain,
-  routerChatWithUsage,
+  mapExecutorResult,
+  supervisorAgent,
 } from '../../src/runtime/index'
 import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
+import {
+  benchProfileModel,
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 
 function must(k: string): string {
   const v = process.env[k]
@@ -47,45 +50,48 @@ const K = Number(process.env.K ?? 3)
 const OFFSET = Number(process.env.OFFSET ?? 0)
 const WORKER_TEMP = Number(process.env.WORKER_TEMP ?? 0.7)
 
-const cfg: RouterConfig = {
+const cfg: BenchRouterTarget = {
   routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
   routerKey: must('TANGLE_API_KEY'),
-  model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+  profile: benchRouterProfile(
+    'humaneval-worker',
+    process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+    { temperature: WORKER_TEMP },
+  ),
+}
+const driverCfg: BenchRouterTarget = {
+  ...cfg,
+  profile: benchRouterProfile(
+    'humaneval-driver',
+    process.env.DRIVER_MODEL ?? benchProfileModel(cfg.profile),
+    { maxTurns: K + 4 },
+  ),
 }
-const driverCfg: RouterConfig = { ...cfg, model: process.env.DRIVER_MODEL ?? cfg.model }
-
-// The driver-LLM brain is the SHARED `routerBrain` (the canonical ToolLoopChat seam) — it forwards
-// usage/costUsd, so this bench's driver arms meter their own inference into the conserved pool.
 
 // ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ──
 function humanEvalWorker(task: HumanEvalTask, label: string): Agent<unknown, unknown> {
-  let artifact: ExecutorResult<unknown> | undefined
-  const inner: Executor<unknown> = {
-    runtime: 'router',
-    async execute(_t, signal) {
-      const res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], {
-        temperature: WORKER_TEMP,
-        ...(signal ? { signal } : {}),
-      })
-      const code = extractCode(res.content)
-      artifact = {
-        outRef: contentAddress(code),
-        out: code,
-        spent: { iterations: 1, tokens: res.usage ?? { input: 0, output: 0 }, usd: res.costUsd ?? 0, ms: 0 },
-      }
-      return artifact
-    },
-    teardown: () => Promise.resolve({ destroyed: true }),
-    resultArtifact: () => {
-      if (!artifact) throw new Error('resultArtifact read before execute')
-      return artifact
-    },
-  }
-  const gated = gateOnDeliverable(inner, {
-    check: async (out) => (await runChecker(task, String(out))).pass === 1,
-    describe: `${task.taskId}: the provided test suite passes`,
+  const profile: AgentProfile = withBenchProfile(cfg.profile, {
+    name: label,
+    systemPrompt: basePrompt(task),
   })
-  const spec: AgentSpec = { profile: { name: label } as AgentProfile, harness: null, executor: gated }
+  const routerFactory = createExecutor({
+    backend: 'router',
+    routerBaseUrl: cfg.routerBaseUrl,
+    routerKey: cfg.routerKey,
+  })
+  const executorFactory = (spec: AgentSpec, ctx: Parameters<typeof routerFactory>[1]) => {
+    const inner = routerFactory(spec, ctx)
+    const mapped = mapExecutorResult(inner, (result) => {
+      const raw = result.out as { content?: unknown }
+      const code = extractCode(typeof raw?.content === 'string' ? raw.content : '')
+      return { outRef: contentAddress(code), out: code }
+    })
+    return gateOnDeliverable(mapped, {
+      check: async (out) => (await runChecker(task, String(out))).pass === 1,
+      describe: `${task.taskId}: the provided test suite passes`,
+    })
+  }
+  const spec: AgentSpec = { profile, harness: null, executorFactory }
   return { name: label, act: async () => '', executorSpec: spec } as Agent<unknown, unknown> & {
     executorSpec: AgentSpec
   }
@@ -113,16 +119,21 @@ async function driveTask(
     spawns += 1
     return w
   }
-  const opts: DriverAgentOptions = {
-    name: `drv-${task.taskId}`,
-    brain: routerBrain(driverCfg),
-    blobs,
-    makeWorkerAgent: makeWorker,
-    perWorker: { maxIterations: 2, maxTokens: 4000 },
-    systemPrompt: driverSystem,
-    maxTurns: K + 4,
-  }
-  const root = driverAgent(opts)
+  const root = supervisorAgent(
+    withBenchProfile(driverCfg.profile, {
+      name: `drv-${task.taskId}`,
+      systemPrompt: driverSystem,
+    }),
+    {
+      router: {
+        routerBaseUrl: driverCfg.routerBaseUrl,
+        routerKey: driverCfg.routerKey,
+      },
+      blobs,
+      makeWorkerAgent: makeWorker,
+      perWorker: { maxIterations: 2, maxTokens: 4000 },
+    },
+  )
   const runId = `he-${task.taskId.replace('/', '-')}`
   const result = await createSupervisor<unknown, unknown>().run(root, basePrompt(task), {
     budget: { maxIterations: 100, maxTokens: 400_000 },
@@ -145,21 +156,31 @@ async function blindTask(task: HumanEvalTask): Promise<boolean> {
   for (let i = 0; i < K; i += 1) {
     // A transient router error is a FAILED attempt, not a crash — the driver arm already types
     // an executor throw into a `down` settlement, so the blind arm must match (fair comparison).
-    let res: { content: string }
+    let content = ''
     try {
-      res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], {
-        temperature: WORKER_TEMP,
-      })
+      const res = await runBenchRouterTurn(
+        {
+          routerBaseUrl: cfg.routerBaseUrl,
+          routerKey: cfg.routerKey,
+          profile: withBenchProfile(cfg.profile, {
+            name: 'humaneval-blind-atom-worker',
+          }),
+        },
+        basePrompt(task),
+      )
+      content = res.finalText
     } catch {
       continue
     }
-    if ((await runChecker(task, extractCode(res.content))).pass === 1) return true
+    if ((await runChecker(task, extractCode(content))).pass === 1) return true
   }
   return false
 }
 
 async function main(): Promise<void> {
-  console.log(`atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${cfg.model} driver=${driverCfg.model}`)
+  console.log(
+    `atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${benchProfileModel(cfg.profile)} driver=${benchProfileModel(driverCfg.profile)}`,
+  )
   const tasks = await loadHumanEval(N, OFFSET)
   const outcomes: TaskOutcome[] = []
   for (const task of tasks) {

package/src/atom-mcp-e2e.mts

@@ -21,7 +21,9 @@ import {
   type Agent,
   type AgentProfile,
   type AgentSpec,
+  collectAgentTurn,
   contentAddress,
+  createExecutor,
   createExecutorRegistry,
   createSupervisor,
   type Executor,
@@ -31,6 +33,7 @@ import {
   InMemorySpawnJournal,
   runInWorkspace,
   type Scope,
+  streamAgentTurn,
   type Workspace,
 } from '../../src/runtime/index'
 import { asAuthoredProfile } from '../../src/runtime/supervise/authoring'
@@ -83,19 +86,30 @@ async function bridgeChat(opts: {
   cwd?: string
   mcpUrl?: string
 }): Promise<string> {
-  const r = await fetch(`${BRIDGE}/chat/completions`, {
-    method: 'POST',
-    headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' },
-    body: JSON.stringify({
-      model: MODEL,
-      messages: opts.messages,
-      ...(opts.cwd ? { cwd: opts.cwd } : {}),
-      ...(opts.mcpUrl ? { mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } } } : {}),
-    }),
+  if (!BEARER) throw new Error('TANGLE_API_KEY is required')
+  const profile: AgentProfile = {
+    name: opts.mcpUrl ? 'atom-mcp-supervisor-turn' : 'atom-mcp-worker-turn',
+    model: { default: MODEL },
+    ...(opts.mcpUrl
+      ? { mcp: { coordination: { transport: 'http', url: opts.mcpUrl } } }
+      : {}),
+  }
+  const factory = createExecutor({
+    backend: 'bridge',
+    bridgeUrl: BRIDGE.replace(/\/v1$/u, ''),
+    bridgeBearer: BEARER,
+    ...(opts.cwd ? { cwd: opts.cwd } : {}),
   })
-  if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})`
-  const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  return j.choices?.[0]?.message?.content ?? ''
+  const turn = await collectAgentTurn(
+    streamAgentTurn(
+      { kind: 'executor', factory, profile },
+      opts.messages.map((message) => message.content).join('\n\n'),
+    ),
+  )
+  if (turn.status !== 'completed') {
+    throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`)
+  }
+  return turn.finalText
 }
 
 const transcripts: Array<{ who: string; said: string; delivered?: boolean }> = []
@@ -109,7 +123,7 @@ function makeWorker(rawProfile: unknown, ws: Workspace, n: number): Agent<unknow
   const inner: Executor<unknown> = {
     runtime: 'router',
     async execute() {
-      const sys = p?.systemPrompt ?? TASK
+      const sys = p?.prompt.systemPrompt ?? TASK
       const run = await runInWorkspace(
         ws,
         async (cwd) => {

package/src/benchmarks/appworld.test.mts

@@ -8,7 +8,13 @@
  */
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { appworldSolutionOutput, createAppWorldAdapter } from './appworld'
+import {
+  appworldReactResultWithUsage,
+  appworldReactUsageEvent,
+  appworldSolutionOutput,
+  appworldToolLoopClient,
+  createAppWorldAdapter,
+} from './appworld'
 
 type Events = Parameters<typeof appworldSolutionOutput.parse>[0]
 const stream = (text: string): Events => [{ data: { finalText: text } }] as unknown as Events
@@ -49,3 +55,115 @@ test('loadTasks either enumerates live engine rows or FAILS LOUD without fabrica
     assert.match((err as Error).message, /appworld driver failed|appworld import failed/)
   }
 })
+
+test('successful react episode survives unknown catalog dollars without fabricating billed cost', () => {
+  const result = appworldReactResultWithUsage(
+    { success: true, passes: 3, fails: 0, num_tests: 3 },
+    {
+      input: 120,
+      output: 30,
+      tokensKnown: true,
+      costUsd: 0.0042,
+      usdKnown: false,
+    },
+    2,
+    'completed task',
+  )
+
+  assert.equal(result.success, true)
+  assert.equal(result.input_tokens, 120)
+  assert.equal(result.output_tokens, 30)
+  assert.equal(result.cost_usd, undefined)
+  const event = appworldReactUsageEvent(result, 'deepseek-v4-flash')
+  assert.deepEqual(event?.data, {
+    model: 'deepseek-v4-flash',
+    tokensIn: 120,
+    tokensOut: 30,
+  })
+  assert.equal(Object.hasOwn(event?.data ?? {}, 'costUsd'), false)
+})
+
+interface TestToolLoopBox {
+  streamPrompt(
+    prompt: string,
+    opts?: { signal?: AbortSignal },
+  ): AsyncGenerator<Record<string, unknown>>
+}
+
+async function testToolLoopBox(client: unknown): Promise<TestToolLoopBox> {
+  return (client as { create(): Promise<TestToolLoopBox> }).create()
+}
+
+async function drain(stream: AsyncGenerator<Record<string, unknown>>): Promise<void> {
+  for await (const _event of stream) {
+    // drain
+  }
+}
+
+test('react client refuses an already-aborted round before a Python session or model call', async () => {
+  let sessionCalls = 0
+  let modelCalls = 0
+  const controller = new AbortController()
+  controller.abort(new Error('already stopped'))
+  const client = appworldToolLoopClient({
+    model: 'offline-model',
+    routerBaseUrl: 'https://router.invalid',
+    routerKey: 'offline',
+    runWorldSession: async () => {
+      sessionCalls += 1
+      throw new Error('unexpected world session')
+    },
+    complete: async () => {
+      modelCalls += 1
+      return {}
+    },
+  })
+  const box = await testToolLoopBox(client)
+
+  await assert.rejects(
+    drain(box.streamPrompt('@appworld-react task-1 test_normal\n', { signal: controller.signal })),
+    /already stopped/,
+  )
+  assert.equal(sessionCalls, 0)
+  assert.equal(modelCalls, 0)
+})
+
+test('react client threads late abort to both the world session and Router call', async () => {
+  const controller = new AbortController()
+  let sessionSignal: AbortSignal | undefined
+  let modelSignal: AbortSignal | undefined
+  let sessionStopped = false
+  const client = appworldToolLoopClient({
+    model: 'offline-model',
+    routerBaseUrl: 'https://router.invalid',
+    routerKey: 'offline',
+    runWorldSession: async (_taskId, _split, signal, fn) => {
+      sessionSignal = signal
+      try {
+        return await fn(async () => ({ success: true, num_tests: 1, passes: 1 }), 'offline task')
+      } finally {
+        sessionStopped = signal.aborted
+      }
+    },
+    complete: async (_body, request) => {
+      modelSignal = request?.signal
+      return new Promise((_resolve, reject) => {
+        request?.signal?.addEventListener(
+          'abort',
+          () => reject(request.signal?.reason ?? new Error('aborted')),
+          { once: true },
+        )
+      })
+    },
+  })
+  const box = await testToolLoopBox(client)
+  const running = drain(
+    box.streamPrompt('@appworld-react task-1 test_normal\n', { signal: controller.signal }),
+  )
+  setTimeout(() => controller.abort(new Error('late stop')), 0)
+
+  await assert.rejects(running, /aborted|late stop/i)
+  assert.equal(sessionSignal, controller.signal)
+  assert.equal(modelSignal?.aborted, true)
+  assert.equal(sessionStopped, true)
+})

package/src/benchmarks/appworld.ts

@@ -22,7 +22,13 @@
 import { spawn } from 'node:child_process'
 import { join } from 'node:path'
 import { createInterface } from 'node:readline'
-import { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/kernel'
+import {
+  collectAgentTurn,
+  createExecutor,
+  type OutputAdapter,
+  streamAgentTurn,
+  type ToolSpec,
+} from '@tangle-network/agent-runtime/kernel'
 import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
@@ -171,7 +177,7 @@ export function createAppWorldAdapter(): BenchmarkAdapter {
 
 /**
  * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is
- * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —
+ * Runtime's profile-bound `router-tools` executor with one tool —
  * `execute_python` — bound to a persistent AppWorld world session. The driver's
  * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,
  * evaluate → verdict); every inference turn, the metering, and the typed
@@ -190,7 +196,7 @@ export function createAppWorldAdapter(): BenchmarkAdapter {
  * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.
  */
 
-interface ReactResult {
+export interface ReactResult {
   success?: boolean
   passes?: number
   fails?: number
@@ -199,9 +205,61 @@ interface ReactResult {
   turns?: number
   input_tokens?: number
   output_tokens?: number
+  cost_usd?: number
   transcript?: string
 }
 
+interface ReactRuntimeUsage {
+  input: number
+  output: number
+  costUsd?: number
+  tokensKnown?: boolean
+  usdKnown?: boolean
+}
+
+/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.
+ * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without
+ * discarding the episode's task evidence. */
+export function appworldReactResultWithUsage(
+  verdict: ReactResult,
+  usage: ReactRuntimeUsage,
+  turns: number | undefined,
+  transcript: string,
+): ReactResult {
+  return {
+    ...verdict,
+    ...(turns !== undefined ? { turns } : {}),
+    ...(usage.tokensKnown === false
+      ? {}
+      : { input_tokens: usage.input, output_tokens: usage.output }),
+    ...(usage.usdKnown === false || usage.costUsd === undefined
+      ? {}
+      : { cost_usd: usage.costUsd }),
+    transcript,
+  }
+}
+
+/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */
+export function appworldReactUsageEvent(
+  result: ReactResult,
+  model: string,
+): { type: 'llm_call'; data: Record<string, unknown> } | undefined {
+  const hasTokens =
+    typeof result.input_tokens === 'number' && typeof result.output_tokens === 'number'
+  const hasCost = typeof result.cost_usd === 'number'
+  if (!hasTokens && !hasCost) return undefined
+  return {
+    type: 'llm_call',
+    data: {
+      model,
+      ...(hasTokens
+        ? { tokensIn: result.input_tokens, tokensOut: result.output_tokens }
+        : {}),
+      ...(hasCost ? { costUsd: result.cost_usd } : {}),
+    },
+  }
+}
+
 const REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/
 
 const SESSION_SYSTEM = [
@@ -231,11 +289,17 @@ const EXECUTE_TOOL: ToolSpec = {
 async function withWorldSession<T>(
   taskId: string,
   split: string,
+  signal: AbortSignal,
   fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,
 ): Promise<T> {
+  signal.throwIfAborted()
   const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {
     cwd: benchRoot,
   })
+  const stopChild = (): void => {
+    if (!child.killed) child.kill('SIGTERM')
+  }
+  signal.addEventListener('abort', stopChild, { once: true })
   const rl = createInterface({ input: child.stdout })
   const pending: Array<(line: string) => void> = []
   const backlog: string[] = []
@@ -250,6 +314,10 @@ async function withWorldSession<T>(
   })
   const nextLine = (timeoutMs: number): Promise<string> =>
     new Promise((resolve, reject) => {
+      if (signal.aborted) {
+        reject(signal.reason)
+        return
+      }
       const fromBacklog = backlog.shift()
       if (fromBacklog !== undefined) return resolve(fromBacklog)
       const t = setTimeout(
@@ -260,14 +328,29 @@ async function withWorldSession<T>(
       // cap) — remove it on the resolve path.
       const onExit = (code: number | null): void => {
         clearTimeout(t)
-        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))
+        signal.removeEventListener('abort', onAbort)
+        reject(
+          signal.aborted
+            ? signal.reason
+            : new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`),
+        )
       }
-      pending.push((l) => {
+      const onAbort = (): void => {
         clearTimeout(t)
         child.removeListener('exit', onExit)
-        resolve(l)
-      })
+        const index = pending.indexOf(onLine)
+        if (index >= 0) pending.splice(index, 1)
+        reject(signal.reason)
+      }
+      const onLine = (line: string): void => {
+        clearTimeout(t)
+        child.removeListener('exit', onExit)
+        signal.removeEventListener('abort', onAbort)
+        resolve(line)
+      }
+      pending.push(onLine)
       child.once('exit', onExit)
+      signal.addEventListener('abort', onAbort, { once: true })
     })
   try {
     const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }
@@ -280,26 +363,43 @@ async function withWorldSession<T>(
     }
     return await fn(call, ready.instruction ?? '')
   } finally {
+    signal.removeEventListener('abort', stopChild)
     child.stdin.end()
-    child.kill('SIGTERM')
+    stopChild()
   }
 }
 
-/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+type AppWorldWorldSession = typeof withWorldSession
+type AppWorldComplete = (
+  body: Record<string, unknown>,
+  request?: {
+    readonly headers: Readonly<Record<string, string>>
+    readonly signal?: AbortSignal
+  },
+) => Promise<unknown>
+
+/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */
 export function appworldToolLoopClient(cfg: {
   model: string
   routerBaseUrl: string
   routerKey: string
   maxTurns?: number
+  /** Offline-test seam; production always uses the Python AppWorld session above. */
+  runWorldSession?: AppWorldWorldSession
+  /** Offline-test seam; production uses Runtime's Router HTTP transport. */
+  complete?: AppWorldComplete
 }): unknown {
   const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)
+  const runWorldSession = cfg.runWorldSession ?? withWorldSession
   let seq = 0
   return {
     async create() {
       const id = `appworld-toolloop-${seq++}`
       return {
         id,
-        async *streamPrompt(prompt: string) {
+        async *streamPrompt(prompt: string, promptOpts?: { signal?: AbortSignal }) {
+          const signal = promptOpts?.signal ?? new AbortController().signal
+          signal.throwIfAborted()
           const m = prompt.match(REACT_HEADER)
           if (!m) {
             throw new Error(
@@ -308,42 +408,65 @@ export function appworldToolLoopClient(cfg: {
           }
           const [, taskId, split] = m
           const directive = prompt.replace(REACT_HEADER, '').trim()
-          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {
+          const out = await runWorldSession(taskId as string, split as string, signal, async (call, instruction) => {
             const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM
-            const loop = await routerToolLoop(
-              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
-              system,
-              `Task: ${instruction}`,
-              [EXECUTE_TOOL],
-              async (name, args) => {
+            const transcriptSteps: Array<{ args: string; result: string }> = []
+            const profile = {
+              name: 'appworld-react-worker',
+              harness: 'cli-base' as const,
+              model: {
+                provider: 'tangle-router',
+                default: cfg.model,
+                metadata: { maxTurns },
+              },
+              prompt: { systemPrompt: system },
+              tools: { execute_python: true },
+            }
+            const factory = createExecutor({
+              backend: 'router-tools',
+              routerBaseUrl: cfg.routerBaseUrl,
+              routerKey: cfg.routerKey,
+              ...(cfg.complete ? { complete: cfg.complete } : {}),
+              tools: [EXECUTE_TOOL],
+              executeToolCall: async (name, args) => {
                 if (name !== 'execute_python') return `error: unknown tool ${name}`
                 const res = await call({ op: 'execute', code: String(args.code ?? '') })
                 const done = res.task_completed === true
-                return `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`
+                const result = `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`
+                transcriptSteps.push({ args: JSON.stringify(args), result })
+                return result
               },
-              { maxTurns },
+            })
+            const loop = await collectAgentTurn(
+              streamAgentTurn(
+                { kind: 'executor', factory, profile },
+                `Task: ${instruction}`,
+                { signal },
+              ),
             )
+            if (loop.status !== 'completed') {
+              throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`)
+            }
             const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult
-            const transcript = loop.toolTrace
+            const transcript = transcriptSteps
               .slice(-3)
               .map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`)
               .join('\n---\n')
               .slice(0, 1600)
-            return {
-              ...verdict,
-              turns: loop.turns,
-              input_tokens: loop.usage.input,
-              output_tokens: loop.usage.output,
+            const finalEvent = loop.events.at(-1)
+            const resultMetadata =
+              finalEvent?.type === 'final' && finalEvent.metadata?.result
+                ? (finalEvent.metadata.result as { spent?: { iterations?: number } })
+                : undefined
+            return appworldReactResultWithUsage(
+              verdict,
+              loop.usage,
+              resultMetadata?.spent?.iterations,
               transcript,
-            } satisfies ReactResult
+            )
           })
-          // Real usage from the episode — flat llm_call so the kernel meters it.
-          if (out.input_tokens || out.output_tokens) {
-            yield {
-              type: 'llm_call',
-              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },
-            }
-          }
+          const usageEvent = appworldReactUsageEvent(out, cfg.model)
+          if (usageEvent) yield usageEvent
           yield { type: 'result', data: { finalText: JSON.stringify(out) } }
         },
         async delete() {},