@tangle-network/agent-bench 0.10.0 → 0.11.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.10.0",
+  "version": "0.11.1",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.178.0 <0.179.0",
+    "@tangle-network/agent-eval": ">=0.179.0 <0.180.0",
     "@tangle-network/agent-interface": "^2.6.0",
-    "@tangle-network/agent-knowledge": "^15.0.1",
-    "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.203.0"
+    "@tangle-network/agent-knowledge": "^15.0.2",
+    "@tangle-network/sandbox": ">=0.36.4 <0.39.0",
+    "@tangle-network/agent-runtime": "^0.204.1"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/verify-packed-consumer.mjs

@@ -165,6 +165,51 @@ writeFileSync(
     path.join(consumerDir, 'index.ts'),
     "import { createSweBenchAdapter, executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, runStagedJudge, StagedJudgeError, type BenchmarkAdapter, type JudgeArtifactReceipt, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierDockerConnection, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst captureAdapter: BenchmarkAdapter = createSweBenchAdapter({ captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination: `/tmp/${taskId}/${attemptSequence}` }) })\nconst receipt = undefined as JudgeArtifactReceipt | undefined\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nconst dockerConnection = undefined as PierDockerConnection | undefined\nvoid adapter\nvoid captureAdapter\nvoid receipt\nvoid staged\nvoid trial\nvoid controller\nvoid dockerConnection\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\nvoid runStagedJudge\nvoid StagedJudgeError\n",
   )
+  await writeFile(
+    path.join(consumerDir, 'managed-execution.ts'),
+    `import assert from 'node:assert/strict'
+import { runBenchmarks, type BenchExecution, type BenchExecutionContext, type BenchPromptResult } from '@tangle-network/agent-bench'
+
+const sessions: Array<string | undefined> = []
+let closed = 0
+const execute: BenchExecution = async (context: BenchExecutionContext) => {
+  await context.run.start(context.prompt)
+  await context.run.resume('corrected')
+}
+const report = await runBenchmarks({
+  benchmarks: ['fixture'], cells: [{ label: 'worker', model: 'fixture' }],
+  routerBaseUrl: 'unused', routerKey: 'unused', verifyJudge: false, execute,
+  resolveAdapter: () => ({
+    name: 'fixture', preflight: async () => {},
+    loadTasks: async () => [{ id: 'task', prompt: 'initial' }],
+    goldArtifact: async () => undefined,
+    judge: async (_task, artifact) => ({ resolved: artifact === 'corrected', score: artifact === 'corrected' ? 1 : 0 }),
+  }),
+  resolveClient: () => ({
+    criuStatus: async () => ({ available: false }),
+    create: async () => ({
+      id: 'packed-fixture',
+      async *streamPrompt(prompt: string, options?: { sessionId?: string }) {
+        sessions.push(options?.sessionId)
+        yield { type: 'llm_call', data: { tokensIn: 2, tokensOut: 1, costUsd: 0.01 } }
+        yield { type: 'result', data: { finalText: prompt, success: true, status: 'success' } }
+        yield { type: 'done', data: { outcome: { type: 'completed' } } }
+      },
+      delete: async () => { closed += 1 },
+    }),
+  }) as never,
+})
+const prompts: readonly BenchPromptResult[] = report.perTask[0]?.prompts ?? []
+assert.equal(report.perTask[0]?.resolved, true)
+assert.equal(prompts.length, 2)
+assert.equal(prompts[1]?.method, 'resume')
+assert.equal(prompts[1]?.prompt, 'corrected')
+assert.equal(sessions[0], sessions[1])
+assert.ok(sessions[0])
+assert.equal(closed, 1)
+assert.deepEqual(report.perTask[0]?.usage, { input: 4, output: 2, costUsd: 0.02 })
+`,
+  )
   await writeFile(
     path.join(consumerDir, 'index.mjs'),
     "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
@@ -189,7 +234,7 @@ if (!score.resolved || score.score !== 1) {
     `${JSON.stringify(
       {
         compilerOptions: publicTsconfig.compilerOptions,
-        files: ['index.ts'],
+        files: ['index.ts', 'managed-execution.ts'],
       },
       null,
       2,
@@ -255,6 +300,7 @@ for name in sorted(expected):
     throw new Error(`expected TypeScript ${TYPESCRIPT_6}, received ${typescript6.stdout.trim()}`)
   }
   await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir)
+  await run('npm', ['exec', '--', 'tsx', 'managed-execution.ts'], consumerDir)
   const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench')
   const prepared = await run(
     'npm',

package/src/index.ts

@@ -64,6 +64,9 @@ export {
   runBenchmarks,
   printBenchmarksReport,
   type BenchCell,
+  type BenchExecution,
+  type BenchExecutionContext,
+  type BenchPromptResult,
   type BenchShot,
   type BenchShotResult,
   type BenchCellTaskResult,

package/src/run-benchmarks.test.mts

@@ -5,7 +5,7 @@
  */
 import assert from 'node:assert/strict'
 import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
-import { runBenchmarks, type BenchShot } from './run-benchmarks'
+import { runBenchmarks, type BenchShot, type BenchExecution, type BenchExecutionContext } from './run-benchmarks'
 
 function stubAdapter(name: string, n: number): BenchmarkAdapter {
   const tasks: BenchTask[] = Array.from({ length: n }, (_, i) => ({
@@ -47,6 +47,7 @@ const shot: BenchShot = async ({ adapter, task, cell }) => {
 }
 
 async function main(): Promise<void> {
+  await managedExecutionProof()
   // Matrix: 2 benchmarks × 3 cells × 4 tasks = 24 shots.
   const report = await runBenchmarks({
     benchmarks: ['alpha', 'beta'],
@@ -391,4 +392,180 @@ async function main(): Promise<void> {
   console.log('run-benchmarks.test: OK (24-shot matrix, subset, reps, unavailable-skip, judge self-check, guards)')
 }
 
+async function managedExecutionProof(): Promise<void> {
+  function fixture(options: { failure?: 'stream' | 'abort'; signal?: AbortSignal; onSecond?: () => void } = {}) {
+    const operations: string[] = []
+    const requests: Array<{ prompt: string; sessionId?: string }> = []
+    let creates = 0
+    let grades = 0
+    const client = {
+      async create() {
+        creates += 1
+        let patch = 'WRONG'
+        let turns = 0
+        return {
+          id: `managed-box-${creates}`,
+          async exec(command: string, opts?: { sessionId?: string }) {
+            operations.push(command)
+            assert.ok(opts?.sessionId, 'working checks and extraction address the worker session')
+            return { exitCode: 0, stdout: command === 'extract' ? patch : 'working check: repair the missing branch', stderr: '' }
+          },
+          async *streamPrompt(prompt: string, opts?: { sessionId?: string; signal?: AbortSignal }) {
+            turns += 1
+            requests.push({ prompt, sessionId: opts?.sessionId })
+            operations.push(`prompt:${turns}`)
+            // A repeated receipt id across prompts must not collapse paid calls.
+            yield { type: 'llm_call', data: { id: 'same-receipt-id', tokensIn: 11, tokensOut: 3, costUsd: 0.02 } }
+            if (turns === 2) {
+              options.onSecond?.()
+              if (options.failure === 'stream') throw new Error('second prompt disconnected')
+              if (options.failure === 'abort') {
+                assert.equal(opts?.signal?.aborted, true)
+                throw Object.assign(new Error('cancelled second prompt'), { name: 'AbortError' })
+              }
+            }
+            if (prompt.includes('repair the missing branch')) patch = 'PATCH'
+            yield { type: 'result', data: { finalText: patch, success: true, status: 'success' } }
+            yield { type: 'done', data: { outcome: { type: 'completed' } } }
+          },
+          async delete() { operations.push('delete') },
+        }
+      },
+      async criuStatus() { return { available: false } },
+    }
+    const adapter: BenchmarkAdapter = {
+      name: 'managed',
+      preflight: async () => {},
+      loadTasks: async () => [{ id: 'task', prompt: 'fix the branch', metadata: { gold: 'PRIVATE FINAL ORACLE' } }],
+      boxSetup: () => ({ command: 'setup' }),
+      boxExtract: () => ({ command: 'extract' }),
+      goldArtifact: async () => 'PATCH',
+      judge: async (_task, artifact) => {
+        grades += 1
+        operations.push('judge')
+        return { resolved: artifact === 'PATCH', score: artifact === 'PATCH' ? 1 : 0 }
+      },
+    }
+    return {
+      operations, requests, get creates() { return creates }, get grades() { return grades },
+      run: (execute?: BenchExecution, loopAttempts = 1) => runBenchmarks({
+        benchmarks: ['managed'], cells: [{ label: 'worker', model: 'model', backend: 'sandbox' }],
+        routerBaseUrl: 'unused', routerKey: 'unused', verifyJudge: false, loopAttempts,
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(execute ? { execute } : {}),
+        resolveAdapter: () => adapter, resolveClient: () => client as never,
+      }),
+    }
+  }
+  const correct: BenchExecution = async (context) => {
+    assert.deepEqual(Object.keys(context).sort(), ['attempt', 'benchmark', 'profile', 'prompt', 'run', 'signal', 'taskId'])
+    assert.equal('close' in context.run, false)
+    assert.equal(context.profile.model?.default, 'model')
+    const first = await context.run.start(context.prompt)
+    assert.equal(first.out, 'WRONG')
+    const feedback = await context.run.box.exec('working-check', { sessionId: context.run.sessionId })
+    await context.run.resume(feedback.stdout)
+  }
+  const enabled = fixture()
+  const report = await enabled.run(correct)
+  assert.equal(report.perTask[0]?.resolved, true)
+  assert.deepEqual(enabled.operations, ['setup', 'prompt:1', 'working-check', 'prompt:2', 'extract', 'delete', 'judge'])
+  assert.equal(enabled.creates, 1)
+  assert.equal(enabled.grades, 1)
+  assert.equal(enabled.requests[0]?.sessionId, enabled.requests[1]?.sessionId)
+  assert.equal(enabled.requests[1]?.prompt, 'working check: repair the missing branch')
+  assert.deepEqual(report.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 })
+  assert.deepEqual(report.perTask[0]?.prompts?.map((p) => [p.attempt, p.index, p.method, p.prompt]), [
+    [1, 0, 'start', 'fix the branch'], [1, 1, 'resume', 'working check: repair the missing branch'],
+  ])
+  assert.equal(report.perTask[0]?.events?.length, 6)
+  const disabled = fixture()
+  const withheld = await disabled.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    await run.resume('try again without a correction')
+  })
+  assert.equal(withheld.perTask[0]?.resolved, false, 'withholding the correction prevents the scripted repair')
+  assert.deepEqual(withheld.perTask[0]?.usage, report.perTask[0]?.usage, 'control spends the same scripted resources')
+  const defaultRun = await fixture().run()
+  assert.equal(defaultRun.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(defaultRun.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+
+  for (const failure of ['stream', 'abort'] as const) {
+    const controller = new AbortController()
+    const broken = fixture({ failure, signal: controller.signal, onSecond: () => { if (failure === 'abort') controller.abort() } })
+    const failed = await broken.run(correct)
+    const row = failed.perTask[0]!
+    assert.equal(row.ok, false)
+    assert.equal(row.measurement, 'unavailable')
+    assert.equal(row.prompts?.length, 2)
+    assert.equal(row.events?.length, 4)
+    assert.deepEqual(row.usage, { input: 22, output: 6, costUsd: 0.04, tokensKnown: false, usdKnown: false })
+    assert.equal(row.prompts?.[0]?.usage.tokensKnown, undefined)
+    assert.equal(row.prompts?.[1]?.usage.tokensKnown, false)
+    assert.equal(broken.operations.filter((op) => op === 'delete').length, 1)
+    assert.equal(broken.operations.includes('extract'), false)
+  }
+  const policyFailure = fixture()
+  const policyFailed = await policyFailure.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    throw new Error('policy failed after paid work')
+  })
+  assert.equal(policyFailed.perTask[0]?.ok, false)
+  assert.deepEqual(policyFailed.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+  assert.match(policyFailed.perTask[0]?.detail ?? '', /policy failed/)
+  assert.equal(policyFailure.operations.filter((op) => op === 'delete').length, 1)
+
+  const policyAbortController = new AbortController()
+  const policyAbort = fixture({ signal: policyAbortController.signal })
+  const abortedPolicy = await policyAbort.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    policyAbortController.abort()
+    await new Promise<void>(() => {})
+  })
+  assert.equal(abortedPolicy.perTask[0]?.ok, false, 'cancellation stops waiting for policy work')
+  assert.equal(abortedPolicy.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(abortedPolicy.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+  assert.equal(policyAbort.operations.filter((op) => op === 'delete').length, 1)
+
+  const looped = fixture()
+  const retried = await looped.run(async ({ run, prompt, attempt }) => {
+    await run.start(prompt)
+    await run.resume(attempt === 2 ? 'repair the missing branch' : 'check again')
+  }, 2)
+  assert.equal(retried.perTask[0]?.resolved, true)
+  assert.equal(looped.creates, 2)
+  assert.deepEqual(retried.perTask[0]?.prompts?.map((p) => [p.attempt, p.index]), [[1, 0], [1, 1], [2, 0], [2, 1]])
+  assert.deepEqual(retried.perTask[0]?.usage, { input: 44, output: 12, costUsd: 0.08 })
+  assert.equal(looped.operations.filter((op) => op === 'setup').length, 2)
+  assert.equal(looped.operations.filter((op) => op === 'extract').length, 2)
+  assert.equal(looped.operations.filter((op) => op === 'delete').length, 2)
+
+  const unawaited = fixture()
+  const awaitedByOwner = await unawaited.run(async ({ run, prompt }) => { void run.start(prompt) })
+  assert.equal(awaitedByOwner.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(unawaited.operations, ['setup', 'prompt:1', 'extract', 'delete', 'judge'])
+  const overlapping = fixture()
+  const overlap = await overlapping.run(async ({ run, prompt }) => {
+    const first = run.start(prompt)
+    assert.throws(() => run.resume('overlap'), /sequential/)
+    await first
+  })
+  assert.equal(overlap.perTask[0]?.ok, false)
+  assert.equal(overlapping.operations.includes('extract'), false)
+  let retained: BenchExecutionContext['run'] | undefined
+  await fixture().run(async ({ run, prompt }) => { retained = run; await run.start(prompt) })
+  assert.throws(() => retained!.resume('too late'), /settled/)
+  const skipped = await fixture().run(async () => {})
+  assert.equal(skipped.perTask[0]?.ok, false)
+  assert.equal(skipped.perTask[0]?.prompts?.length, 0)
+  const passthrough: BenchExecution = async () => {}
+  let received: BenchExecution | undefined
+  await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'custom', model: 'm' }], n: 1,
+    routerBaseUrl: 'unused', routerKey: 'unused', resolveAdapter: resolveStub,
+    execute: passthrough, runShot: async ({ execute }) => { received = execute; return { artifact: '', ok: false } },
+  })
+  assert.equal(received, passthrough, 'custom shots choose how to consume the callback')
+}
+
 void main()

package/src/run-benchmarks.ts

@@ -35,6 +35,8 @@ import type {
   AgentRunSpec,
   Deliverable,
   OpenSandboxRunOptions,
+  SandboxRun,
+  TurnResult,
 } from '@tangle-network/agent-runtime/kernel'
 import { openSandboxRun, SandboxRunAbortError, sumSandboxUsage } from '@tangle-network/agent-runtime/kernel'
 import type { SandboxEvent } from '@tangle-network/sandbox'
@@ -61,6 +63,34 @@ export interface BenchCell {
   readonly profile?: AgentProfile
 }
 
+/** Caller-owned work inside a managed benchmark shot. Final grading stays outside this callback. */
+export interface BenchExecutionContext {
+  readonly prompt: string
+  readonly profile: AgentProfile
+  readonly benchmark: string
+  readonly taskId: string
+  readonly attempt: number
+  readonly signal: AbortSignal
+  /** Runtime owns session continuity. Bench owns capture, extraction, and close. */
+  readonly run: Pick<SandboxRun<string>, 'start' | 'resume' | 'box' | 'sessionId'>
+}
+
+export type BenchExecution = (context: BenchExecutionContext) => Promise<void>
+
+/** One submitted sandbox prompt, including partial evidence when its invocation throws. */
+export interface BenchPromptResult {
+  readonly attempt: number
+  readonly index: number
+  readonly method: 'start' | 'resume'
+  readonly prompt: string
+  readonly sessionId?: string
+  readonly outcome?: TurnResult<string>['outcome']
+  readonly events: readonly SandboxEvent[]
+  readonly usage: ReturnType<typeof sumSandboxUsage>
+  readonly readError?: string
+  readonly error?: string
+}
+
 /** A worker's artifact and observed execution evidence, before external grading. */
 export interface BenchShotResult {
   readonly artifact: string
@@ -69,6 +99,7 @@ export interface BenchShotResult {
   /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
   readonly usage?: ReturnType<typeof sumSandboxUsage>
   readonly events?: readonly SandboxEvent[]
+  readonly prompts?: readonly BenchPromptResult[]
   /** Observed dispatch and terminal state, independent of artifact quality. */
   readonly execution?: {
     readonly phase: 'not-started' | 'started' | 'unknown'
@@ -87,6 +118,8 @@ export type BenchShot = (input: {
   readonly prompt?: string
   /** 1-based attempt index for looped runs. */
   readonly attempt?: number
+  /** Custom shots own whether they consume this managed execution callback. */
+  readonly execute?: BenchExecution
   readonly routerBaseUrl: string
   readonly routerKey: string
   /** Optional inference credential for the box; routerKey continues to authorize sandbox control. */
@@ -131,6 +164,8 @@ export interface RunBenchmarksOptions {
   /** Self-verify each benchmark's judge against its gold artifact on the first task before spending
    *  model tokens; a benchmark whose judge rejects its own gold is recorded unavailable. Default true. */
   readonly verifyJudge?: boolean
+  /** Caller policy inside each managed shot, including every refine-loop attempt. */
+  readonly execute?: BenchExecution
   /** Test seam: a deterministic shot runner. Defaults to the `openSandboxRun` leaf. */
   readonly runShot?: BenchShot
   /** Test seam: resolve a benchmark key to an adapter. Defaults to the registry `resolveAdapter`. */
@@ -157,6 +192,7 @@ export interface BenchCellTaskResult {
   readonly usage?: ReturnType<typeof sumSandboxUsage>
   /** Worker events only; benchmark grading remains outside this trace. */
   readonly events?: readonly SandboxEvent[]
+  readonly prompts?: readonly BenchPromptResult[]
 }
 
 export interface BenchLeaderboardRow {
@@ -204,7 +240,7 @@ function finalText(events: readonly SandboxEvent[]): string {
 
 /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable
  *  extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */
-const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
+const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, attempt = 1, execute, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
   signal?.throwIfAborted()
   const client = (resolveClient ?? resolveBenchClient)({
     backend: cell.backend ?? 'router',
@@ -272,12 +308,91 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   }
   let run: Awaited<ReturnType<typeof openSandboxRun<string>>> | undefined
   let result: BenchShotResult = { artifact: '', ok: false }
+  const prompts: BenchPromptResult[] = []
+  const execution: {
+    lastTurn?: TurnResult<string>
+    active?: Promise<TurnResult<string>>
+    accepting: boolean
+    coordinationError?: Error
+  } = { accepting: true }
   try {
     run = await openSandboxRun(client, runOptions, deliverable)
     result = { ...result, execution: { phase: 'unknown', terminalOutcome: 'unknown' } }
-    const turn = await run.start(prompt ?? task.prompt)
+    const managedRun = run
+    const invoke = (method: 'start' | 'resume', input: string): Promise<TurnResult<string>> => {
+      if (!execution.accepting || execution.active) {
+        execution.coordinationError = new Error(!execution.accepting
+          ? 'benchmark execution callback has settled'
+          : 'benchmark execution requires sequential start/resume calls')
+        throw execution.coordinationError
+      }
+      runOptions.signal.throwIfAborted()
+      const index = prompts.length
+      const eventStart = observedEvents.length
+      execution.lastTurn = undefined
+      const capture = async (): Promise<TurnResult<string>> => {
+        let turn: TurnResult<string> | undefined
+        let failure: unknown
+        try {
+          turn = await managedRun[method](input)
+          return turn
+        } catch (error) {
+          failure = error
+          throw error
+        } finally {
+          // The observer captures generic stream/parser failures as well as aborts.
+          const events = observedEvents.slice(eventStart)
+          if (turn) execution.lastTurn = { ...turn, events, outcome: { ...turn.outcome } }
+          let sessionId: string | undefined
+          try { sessionId = managedRun.sessionId } catch { /* Creation may have failed. */ }
+          const readError = turn?.readError ?? (failure instanceof SandboxRunAbortError ? failure.readError : undefined)
+          prompts.push({
+            attempt, index, method, prompt: input,
+            ...(sessionId === undefined ? {} : { sessionId }),
+            ...(turn === undefined ? {} : { outcome: { ...turn.outcome } }),
+            events,
+            usage: turn === undefined
+              ? { ...sumSandboxUsage(events), tokensKnown: false, usdKnown: false }
+              : sumSandboxUsage(events),
+            ...(readError === undefined ? {} : { readError }),
+            ...(turn !== undefined ? {} : { error: failure instanceof Error ? failure.message : String(failure) }),
+          })
+        }
+      }
+      const pending = capture()
+      execution.active = pending
+      // Observe even a caller's unawaited invocation; cleanup must await its capture.
+      void pending.then(() => { execution.active = undefined }, () => { execution.active = undefined })
+      return pending
+    }
+    const context: BenchExecutionContext = {
+      prompt: prompt ?? task.prompt, profile: structuredClone(profile), benchmark: adapter.name, taskId: task.id, attempt,
+      signal: runOptions.signal,
+      run: {
+        start: (input) => invoke('start', input),
+        resume: (input) => invoke('resume', input),
+        get box() { return managedRun.box },
+        get sessionId() { return managedRun.sessionId },
+      },
+    }
+    try {
+      await executeWithSignal(async () => {
+        if (execute) await execute(context)
+        else await context.run.start(context.prompt)
+      }, runOptions.signal)
+    } catch (error) {
+      if (execution.active) controller.abort()
+      throw error
+    } finally {
+      execution.accepting = false
+      await execution.active?.catch(() => undefined)
+    }
+    if (execution.coordinationError) throw execution.coordinationError
+    runOptions.signal.throwIfAborted()
+    const turn = execution.lastTurn
+    if (!turn) throw new Error(prompts.at(-1)?.error ?? 'benchmark execution returned without a completed prompt')
     result = {
-      artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events,
+      artifact: '', ok: false, usage: promptUsage(prompts), events: observedEvents, prompts,
       execution: {
         phase: 'started',
         terminalOutcome: turn.outcome.success ? 'succeeded' : turn.outcome.status === 'failed' ? 'failed' : 'incomplete',
@@ -358,11 +473,12 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
       execution: result.execution,
       artifactAvailable: turn.readError === undefined && boxExtractError === undefined,
       usage: result.usage,
-      events: turn.events,
+      events: observedEvents,
+      prompts,
       ...(detail ? { detail } : {}),
     }
   } catch (err) {
-    const events = err instanceof SandboxRunAbortError ? err.events : observedEvents
+    const events = observedEvents
     result = {
       ...result,
       ok: false,
@@ -372,9 +488,9 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
         terminalOutcome: result.execution?.terminalOutcome ?? 'unknown',
       },
       detail: err instanceof Error ? err.message : String(err),
-      // A thrown capture cannot establish that every paid receipt arrived.
-      usage: { ...sumSandboxUsage(events), tokensKnown: false, usdKnown: false },
+      usage: promptUsage(prompts),
       events,
+      prompts,
     }
   } finally {
     if (timer) clearTimeout(timer)
@@ -388,6 +504,25 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   return result
 }
 
+/** Stop waiting on policy work when cancelled; the caller must also cancel its external effects. */
+async function executeWithSignal(execute: () => Promise<void>, signal: AbortSignal): Promise<void> {
+  signal.throwIfAborted()
+  let onAbort: (() => void) | undefined
+  const aborted = new Promise<never>((_resolve, reject) => {
+    onAbort = () => reject(signal.reason ?? new Error('aborted'))
+    signal.addEventListener('abort', onAbort, { once: true })
+  })
+  try {
+    await Promise.race([Promise.resolve().then(execute), aborted])
+  } finally {
+    if (onAbort) signal.removeEventListener('abort', onAbort)
+  }
+}
+
+function promptUsage(prompts: readonly BenchPromptResult[]): ReturnType<typeof sumSandboxUsage> {
+  return combinedUsage(prompts.map((prompt) => ({ artifact: '', ok: false, usage: prompt.usage })))
+}
+
 function parseMaybeJson(value: string): unknown {
   try {
     return JSON.parse(value) as unknown
@@ -484,6 +619,7 @@ async function loopedShot(
       artifactAvailable: false,
       usage: combinedUsage(pendingShot ? [...completed, { artifact: '', ok: false }] : completed),
       events: completed.flatMap((shot) => shot.events ?? []),
+      prompts: completed.flatMap((shot) => shot.prompts ?? []),
       detail: err instanceof Error ? err.message : String(err),
     }
   }
@@ -509,6 +645,7 @@ async function loopedShot(
     artifactAvailable: shots.get(best.round)?.artifactAvailable,
     usage: combinedUsage([...shots.values()]),
     events: [...shots.values()].flatMap((shot) => shot.events ?? []),
+    prompts: [...shots.values()].flatMap((shot) => shot.prompts ?? []),
     detail: JSON.stringify({
       mode: 'refine-loop',
       attempts: result.rounds.length,
@@ -639,6 +776,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}),
         ...(opts.signal ? { signal: opts.signal } : {}),
         ...(opts.resolveClient ? { resolveClient: opts.resolveClient } : {}),
+        ...(opts.execute ? { execute: opts.execute } : {}),
       }
       invoked = true
       out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
@@ -658,6 +796,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         artifact: out.artifact,
         ...(out.usage === undefined ? {} : { usage: out.usage }),
         ...(out.events === undefined ? {} : { events: out.events }),
+        ...(out.prompts === undefined ? {} : { prompts: out.prompts }),
       }
     } catch (err) {
       // Missing results do not prove that dispatch or paid inference never occurred.
@@ -679,6 +818,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ...(out === undefined ? {} : { artifact: out.artifact }),
         ...(out?.usage === undefined ? {} : { usage: out.usage }),
         ...(out?.events === undefined ? {} : { events: out.events }),
+        ...(out?.prompts === undefined ? {} : { prompts: out.prompts }),
       }
     }
     void index