@tangle-network/agent-bench 0.9.4 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.9.4",
+  "version": "0.11.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.176.0 <0.177.0",
-    "@tangle-network/agent-interface": "^2.5.0",
-    "@tangle-network/agent-knowledge": "^15.0.0",
+    "@tangle-network/agent-eval": ">=0.178.0 <0.179.0",
+    "@tangle-network/agent-interface": "^2.6.0",
+    "@tangle-network/agent-knowledge": "^15.0.1",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.202.0"
+    "@tangle-network/agent-runtime": "^0.203.2"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/run-package-tests.mjs

@@ -1,5 +1,6 @@
 import { execFile } from 'node:child_process'
-import { access, readdir, readFile } from 'node:fs/promises'
+import { access, readdir, readFile, realpath } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
@@ -64,6 +65,15 @@ export async function run(command, args, env = process.env) {
   }
 }
 
+export async function runPythonTests(python, env = process.env) {
+  // Fixture roots must be physical paths; the production boundary rejects symlinked ancestors.
+  const physicalTemp = await realpath(env.TMPDIR || tmpdir())
+  await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'], {
+    ...env,
+    TMPDIR: physicalTemp,
+  })
+}
+
 async function main() {
   const python = path.join(benchDir, '.venv', 'bin', 'python')
   try {
@@ -101,7 +111,7 @@ async function main() {
     await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
   }
 
-  await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
+  await runPythonTests(python)
 
   console.log(
     `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,

package/scripts/run-package-tests.test.mjs

@@ -1,6 +1,9 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { packageNodeTestArgs, resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs'
+import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { packageNodeTestArgs, resolvePackageTestTimeoutMs, run, runPythonTests } from './run-package-tests.mjs'
 
 test('package test concurrency reaches Node without changing selected files', () => {
   const files = ['src/first.test.mts', 'src/second.test.ts']
@@ -45,3 +48,25 @@ test('package test timeout reaches the child process', async () => {
   )
   assert.ok(Date.now() - startedAt < 2_000)
 })
+
+test('Python fixtures use physical temporary paths without relaxing symlink guards', async () => {
+  const root = await mkdtemp(path.join(await realpath(tmpdir()), 'bench-python-temp-'))
+  try {
+    const target = path.join(root, 'physical')
+    const linked = path.join(root, 'linked')
+    const recorded = path.join(root, 'recorded.json')
+    const launcher = path.join(root, 'python-fixture')
+    await mkdir(target)
+    await symlink(target, linked, 'dir')
+    await writeFile(launcher, `#!${process.execPath}
+const fs = require('node:fs'); fs.writeFileSync(process.env.RECORDED, JSON.stringify({ temporary: process.env.TMPDIR, args: process.argv.slice(2) }))
+`, { mode: 0o755 })
+    await runPythonTests(launcher, { ...process.env, TMPDIR: linked, RECORDED: recorded })
+    assert.deepEqual(JSON.parse(await readFile(recorded, 'utf8')), {
+      temporary: target,
+      args: ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'],
+    })
+  } finally {
+    await rm(root, { recursive: true, force: true })
+  }
+})

package/scripts/verify-packed-consumer.mjs

@@ -165,6 +165,51 @@ writeFileSync(
     path.join(consumerDir, 'index.ts'),
     "import { createSweBenchAdapter, executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, runStagedJudge, StagedJudgeError, type BenchmarkAdapter, type JudgeArtifactReceipt, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierDockerConnection, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst captureAdapter: BenchmarkAdapter = createSweBenchAdapter({ captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination: `/tmp/${taskId}/${attemptSequence}` }) })\nconst receipt = undefined as JudgeArtifactReceipt | undefined\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nconst dockerConnection = undefined as PierDockerConnection | undefined\nvoid adapter\nvoid captureAdapter\nvoid receipt\nvoid staged\nvoid trial\nvoid controller\nvoid dockerConnection\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\nvoid runStagedJudge\nvoid StagedJudgeError\n",
   )
+  await writeFile(
+    path.join(consumerDir, 'managed-execution.ts'),
+    `import assert from 'node:assert/strict'
+import { runBenchmarks, type BenchExecution, type BenchExecutionContext, type BenchPromptResult } from '@tangle-network/agent-bench'
+
+const sessions: Array<string | undefined> = []
+let closed = 0
+const execute: BenchExecution = async (context: BenchExecutionContext) => {
+  await context.run.start(context.prompt)
+  await context.run.resume('corrected')
+}
+const report = await runBenchmarks({
+  benchmarks: ['fixture'], cells: [{ label: 'worker', model: 'fixture' }],
+  routerBaseUrl: 'unused', routerKey: 'unused', verifyJudge: false, execute,
+  resolveAdapter: () => ({
+    name: 'fixture', preflight: async () => {},
+    loadTasks: async () => [{ id: 'task', prompt: 'initial' }],
+    goldArtifact: async () => undefined,
+    judge: async (_task, artifact) => ({ resolved: artifact === 'corrected', score: artifact === 'corrected' ? 1 : 0 }),
+  }),
+  resolveClient: () => ({
+    criuStatus: async () => ({ available: false }),
+    create: async () => ({
+      id: 'packed-fixture',
+      async *streamPrompt(prompt: string, options?: { sessionId?: string }) {
+        sessions.push(options?.sessionId)
+        yield { type: 'llm_call', data: { tokensIn: 2, tokensOut: 1, costUsd: 0.01 } }
+        yield { type: 'result', data: { finalText: prompt, success: true, status: 'success' } }
+        yield { type: 'done', data: { outcome: { type: 'completed' } } }
+      },
+      delete: async () => { closed += 1 },
+    }),
+  }) as never,
+})
+const prompts: readonly BenchPromptResult[] = report.perTask[0]?.prompts ?? []
+assert.equal(report.perTask[0]?.resolved, true)
+assert.equal(prompts.length, 2)
+assert.equal(prompts[1]?.method, 'resume')
+assert.equal(prompts[1]?.prompt, 'corrected')
+assert.equal(sessions[0], sessions[1])
+assert.ok(sessions[0])
+assert.equal(closed, 1)
+assert.deepEqual(report.perTask[0]?.usage, { input: 4, output: 2, costUsd: 0.02 })
+`,
+  )
   await writeFile(
     path.join(consumerDir, 'index.mjs'),
     "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
@@ -189,7 +234,7 @@ if (!score.resolved || score.score !== 1) {
     `${JSON.stringify(
       {
         compilerOptions: publicTsconfig.compilerOptions,
-        files: ['index.ts'],
+        files: ['index.ts', 'managed-execution.ts'],
       },
       null,
       2,
@@ -255,6 +300,7 @@ for name in sorted(expected):
     throw new Error(`expected TypeScript ${TYPESCRIPT_6}, received ${typescript6.stdout.trim()}`)
   }
   await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir)
+  await run('npm', ['exec', '--', 'tsx', 'managed-execution.ts'], consumerDir)
   const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench')
   const prepared = await run(
     'npm',

package/src/index.ts

@@ -64,6 +64,9 @@ export {
   runBenchmarks,
   printBenchmarksReport,
   type BenchCell,
+  type BenchExecution,
+  type BenchExecutionContext,
+  type BenchPromptResult,
   type BenchShot,
   type BenchShotResult,
   type BenchCellTaskResult,

package/src/run-benchmarks.test.mts

@@ -5,7 +5,7 @@
  */
 import assert from 'node:assert/strict'
 import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
-import { runBenchmarks, type BenchShot } from './run-benchmarks'
+import { runBenchmarks, type BenchShot, type BenchExecution, type BenchExecutionContext } from './run-benchmarks'
 
 function stubAdapter(name: string, n: number): BenchmarkAdapter {
   const tasks: BenchTask[] = Array.from({ length: n }, (_, i) => ({
@@ -47,6 +47,7 @@ const shot: BenchShot = async ({ adapter, task, cell }) => {
 }
 
 async function main(): Promise<void> {
+  await managedExecutionProof()
   // Matrix: 2 benchmarks × 3 cells × 4 tasks = 24 shots.
   const report = await runBenchmarks({
     benchmarks: ['alpha', 'beta'],
@@ -77,6 +78,22 @@ async function main(): Promise<void> {
   assert.equal(judgeFailure.perTask[0]?.ok, false)
   assert.equal(judgeFailure.perTask[0]?.artifact, 'PATCH', 'a judge outage retains the completed agent artifact')
   assert.equal(judgeFailure.perTask[0]?.usage?.costUsd, 0.04, 'a judge outage retains already incurred usage')
+  assert.equal(judgeFailure.perTask[0]?.measurement, 'unavailable')
+  assert.deepEqual(judgeFailure.perTask[0]?.execution, { phase: 'started', terminalOutcome: 'succeeded' })
+
+  const failedExecution = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'paid-failure', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', n: 1, resolveAdapter: resolveStub,
+    runShot: async () => ({
+      artifact: '', ok: false, artifactAvailable: true,
+      execution: { phase: 'started', terminalOutcome: 'failed' },
+      usage: { input: 23, output: 7, costUsd: 0.04 },
+    }),
+  })
+  assert.equal(failedExecution.perTask[0]?.measurement, 'available')
+  assert.equal(failedExecution.perTask[0]?.resolved, false)
+  assert.equal(failedExecution.rows[0]?.errored, 0, 'a measured paid failure stays in the comparison')
+  assert.equal(failedExecution.perTask[0]?.usage?.input, 23)
 
   const controller = new AbortController()
   let started = 0
@@ -93,6 +110,7 @@ async function main(): Promise<void> {
   })
   assert.equal(started, 1, 'cancellation prevents every queued model call')
   assert.equal(cancelled.perTask.length, 4, 'cancelled work remains visible')
+  assert.equal(cancelled.perTask.filter((row) => row.execution?.phase === 'not-started').length, 3)
   assert.equal(cancelled.perTask[0]?.usage?.costUsd, 0.01)
 
   const row = (b: string, c: string) => report.rows.find((r) => r.benchmark === b && r.cell === c)!
@@ -159,6 +177,20 @@ async function main(): Promise<void> {
     input: 11, output: 3, costUsd: 0.02, tokensKnown: false, usdKnown: false,
   }, 'an unreported retry preserves the measured floor without claiming complete accounting')
 
+  const measuredFailedRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'failed-retries', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ attempt }) => ({
+      artifact: attempt === 1 ? '' : 'WRONG', ok: false, artifactAvailable: attempt === 2,
+      execution: { phase: 'started', terminalOutcome: 'failed' },
+      usage: { input: 11, output: 3, costUsd: 0.02 },
+    }),
+  })
+  assert.equal(measuredFailedRetry.perTask[0]?.artifact, 'WRONG')
+  assert.equal(measuredFailedRetry.perTask[0]?.measurement, 'available')
+  assert.equal(measuredFailedRetry.rows[0]?.errored, 0, 'a measured failed retry stays in the comparison')
+  assert.deepEqual(measuredFailedRetry.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 })
+
   let validAttempts = 0
   const validRetry = await runBenchmarks({
     benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
@@ -212,6 +244,10 @@ async function main(): Promise<void> {
     const order: string[] = []
     let createdOptions: unknown
     let controlCredential: string | undefined
+    let completed = true
+    let text = 'fallback text'
+    let streamThrows = false
+    let setupFails = false
     const fakeClient = {
       async create(options: unknown) {
         createdOptions = options
@@ -219,13 +255,14 @@ async function main(): Promise<void> {
           id: 'box-default-shot',
           async exec(command: string, options?: { sessionId?: string }) {
             order.push(`exec:${command}:streams=${order.filter((x) => x.startsWith('stream:')).length}:session=${options?.sessionId ? 'yes' : 'no'}`)
-            return { exitCode: 0, stdout: command === 'extract-patch' ? 'PATCH' : '', stderr: '' }
+            return { exitCode: setupFails && command === 'setup-repo' ? 1 : 0, stdout: command === 'extract-patch' ? 'PATCH' : '', stderr: '' }
           },
           async *streamPrompt(_prompt: string, options?: { sessionId?: string }) {
             order.push(`stream:session=${options?.sessionId ? 'yes' : 'no'}`)
             yield { type: 'llm_call', data: { tokensIn: 23, tokensOut: 7, costUsd: 0.04 } }
-            yield { type: 'result', data: { finalText: 'fallback text' } }
-            yield { type: 'done', data: { outcome: { type: 'completed' } } }
+            if (streamThrows) throw new Error('stream disconnected')
+            yield { type: 'result', data: { finalText: text, success: completed, status: completed ? 'success' : 'failed' } }
+            yield { type: 'done', data: { outcome: { type: completed ? 'completed' : 'failed' } } }
           },
           async delete() {
             order.push('delete')
@@ -260,6 +297,8 @@ async function main(): Promise<void> {
     assert.equal(boxy.rows[0]!.resolveRate, 1, 'boxExtract artifact is judged instead of fallback text')
     assert.equal(boxy.perTask[0]?.artifact, 'PATCH')
     assert.deepEqual(boxy.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 })
+    assert.deepEqual(boxy.perTask[0]?.execution, { phase: 'started', terminalOutcome: 'succeeded' })
+    assert.equal(boxy.perTask[0]?.measurement, 'available')
     assert.equal(controlCredential, 'sandbox-control-token', 'inference grant never authorizes sandbox control')
     assert.equal((createdOptions as { backend: { model: { apiKey: string } } }).backend.model.apiKey, 'model-grant-token')
     assert.deepEqual(
@@ -267,6 +306,49 @@ async function main(): Promise<void> {
       ['exec:setup-repo:streams=0:session=yes', 'stream:session=yes', 'exec:extract-patch:streams=1:session=yes'],
       'setup runs before the prompt stream, extract runs after the prompt stream, both in the same session',
     )
+    for (const failure of ['stream', 'parser'] as const) {
+      streamThrows = failure === 'stream'
+      const failed = await runBenchmarks({
+        benchmarks: ['boxy'], cells: [{ label: failure, model: 'm', backend: 'sandbox' }],
+        routerBaseUrl: 'x', routerKey: 'x',
+        resolveAdapter: () => ({ ...boxAdapter, output: { parse: () => { throw new Error('parser failed') } } }),
+        resolveClient: () => fakeClient as never,
+      })
+      assert.equal(failed.perTask[0]?.measurement, 'unavailable')
+      assert.equal(failed.perTask[0]?.execution?.phase, 'started')
+      assert.deepEqual(failed.perTask[0]?.usage, {
+        input: 23, output: 7, costUsd: 0.04, tokensKnown: false, usdKnown: false,
+      }, `${failure} failure retains paid receipts without claiming complete accounting`)
+      assert.equal(failed.perTask[0]?.events?.length, failure === 'stream' ? 1 : 3, 'observed events are retained once')
+      assert.match(failed.perTask[0]?.detail ?? '', failure === 'stream' ? /stream disconnected/ : /parser failed/)
+    }
+    streamThrows = false
+    setupFails = true
+    const setupFailure = await runBenchmarks({
+      benchmarks: ['boxy'], cells: [{ label: 'setup-failure', model: 'm', backend: 'sandbox' }],
+      routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: () => boxAdapter,
+      resolveClient: () => fakeClient as never,
+    })
+    assert.deepEqual(setupFailure.perTask[0]?.execution, { phase: 'unknown', terminalOutcome: 'unknown' })
+    assert.equal(setupFailure.perTask[0]?.measurement, 'unavailable')
+    assert.equal(setupFailure.perTask[0]?.usage?.tokensKnown, false, 'no observed receipt does not prove zero usage')
+    assert.equal(setupFailure.perTask[0]?.usage?.usdKnown, false)
+    assert.equal(setupFailure.perTask[0]?.events?.length, 0)
+    setupFails = false
+    for (const success of [false, true]) {
+      completed = success
+      text = ''
+      const empty = await runBenchmarks({
+        benchmarks: ['alpha'], cells: [{ label: 'empty', model: 'm', backend: 'sandbox' }],
+        routerBaseUrl: 'x', routerKey: 'x', n: 1, resolveAdapter: resolveStub,
+        resolveClient: () => fakeClient as never,
+      })
+      assert.equal(empty.perTask[0]?.ok, false)
+      assert.equal(empty.perTask[0]?.measurement, 'available', 'captured empty output is a measurable failure')
+      assert.equal(empty.perTask[0]?.execution?.terminalOutcome, success ? 'succeeded' : 'failed')
+      assert.equal(empty.rows[0]?.errored, 0)
+      assert.equal(empty.perTask[0]?.usage?.input, 23)
+    }
   }
 
   // An unavailable benchmark (preflight throws) is skipped, not fatal; the sweep still runs the rest.
@@ -310,4 +392,180 @@ async function main(): Promise<void> {
   console.log('run-benchmarks.test: OK (24-shot matrix, subset, reps, unavailable-skip, judge self-check, guards)')
 }
 
+async function managedExecutionProof(): Promise<void> {
+  function fixture(options: { failure?: 'stream' | 'abort'; signal?: AbortSignal; onSecond?: () => void } = {}) {
+    const operations: string[] = []
+    const requests: Array<{ prompt: string; sessionId?: string }> = []
+    let creates = 0
+    let grades = 0
+    const client = {
+      async create() {
+        creates += 1
+        let patch = 'WRONG'
+        let turns = 0
+        return {
+          id: `managed-box-${creates}`,
+          async exec(command: string, opts?: { sessionId?: string }) {
+            operations.push(command)
+            assert.ok(opts?.sessionId, 'working checks and extraction address the worker session')
+            return { exitCode: 0, stdout: command === 'extract' ? patch : 'working check: repair the missing branch', stderr: '' }
+          },
+          async *streamPrompt(prompt: string, opts?: { sessionId?: string; signal?: AbortSignal }) {
+            turns += 1
+            requests.push({ prompt, sessionId: opts?.sessionId })
+            operations.push(`prompt:${turns}`)
+            // A repeated receipt id across prompts must not collapse paid calls.
+            yield { type: 'llm_call', data: { id: 'same-receipt-id', tokensIn: 11, tokensOut: 3, costUsd: 0.02 } }
+            if (turns === 2) {
+              options.onSecond?.()
+              if (options.failure === 'stream') throw new Error('second prompt disconnected')
+              if (options.failure === 'abort') {
+                assert.equal(opts?.signal?.aborted, true)
+                throw Object.assign(new Error('cancelled second prompt'), { name: 'AbortError' })
+              }
+            }
+            if (prompt.includes('repair the missing branch')) patch = 'PATCH'
+            yield { type: 'result', data: { finalText: patch, success: true, status: 'success' } }
+            yield { type: 'done', data: { outcome: { type: 'completed' } } }
+          },
+          async delete() { operations.push('delete') },
+        }
+      },
+      async criuStatus() { return { available: false } },
+    }
+    const adapter: BenchmarkAdapter = {
+      name: 'managed',
+      preflight: async () => {},
+      loadTasks: async () => [{ id: 'task', prompt: 'fix the branch', metadata: { gold: 'PRIVATE FINAL ORACLE' } }],
+      boxSetup: () => ({ command: 'setup' }),
+      boxExtract: () => ({ command: 'extract' }),
+      goldArtifact: async () => 'PATCH',
+      judge: async (_task, artifact) => {
+        grades += 1
+        operations.push('judge')
+        return { resolved: artifact === 'PATCH', score: artifact === 'PATCH' ? 1 : 0 }
+      },
+    }
+    return {
+      operations, requests, get creates() { return creates }, get grades() { return grades },
+      run: (execute?: BenchExecution, loopAttempts = 1) => runBenchmarks({
+        benchmarks: ['managed'], cells: [{ label: 'worker', model: 'model', backend: 'sandbox' }],
+        routerBaseUrl: 'unused', routerKey: 'unused', verifyJudge: false, loopAttempts,
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(execute ? { execute } : {}),
+        resolveAdapter: () => adapter, resolveClient: () => client as never,
+      }),
+    }
+  }
+  const correct: BenchExecution = async (context) => {
+    assert.deepEqual(Object.keys(context).sort(), ['attempt', 'benchmark', 'profile', 'prompt', 'run', 'signal', 'taskId'])
+    assert.equal('close' in context.run, false)
+    assert.equal(context.profile.model?.default, 'model')
+    const first = await context.run.start(context.prompt)
+    assert.equal(first.out, 'WRONG')
+    const feedback = await context.run.box.exec('working-check', { sessionId: context.run.sessionId })
+    await context.run.resume(feedback.stdout)
+  }
+  const enabled = fixture()
+  const report = await enabled.run(correct)
+  assert.equal(report.perTask[0]?.resolved, true)
+  assert.deepEqual(enabled.operations, ['setup', 'prompt:1', 'working-check', 'prompt:2', 'extract', 'delete', 'judge'])
+  assert.equal(enabled.creates, 1)
+  assert.equal(enabled.grades, 1)
+  assert.equal(enabled.requests[0]?.sessionId, enabled.requests[1]?.sessionId)
+  assert.equal(enabled.requests[1]?.prompt, 'working check: repair the missing branch')
+  assert.deepEqual(report.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 })
+  assert.deepEqual(report.perTask[0]?.prompts?.map((p) => [p.attempt, p.index, p.method, p.prompt]), [
+    [1, 0, 'start', 'fix the branch'], [1, 1, 'resume', 'working check: repair the missing branch'],
+  ])
+  assert.equal(report.perTask[0]?.events?.length, 6)
+  const disabled = fixture()
+  const withheld = await disabled.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    await run.resume('try again without a correction')
+  })
+  assert.equal(withheld.perTask[0]?.resolved, false, 'withholding the correction prevents the scripted repair')
+  assert.deepEqual(withheld.perTask[0]?.usage, report.perTask[0]?.usage, 'control spends the same scripted resources')
+  const defaultRun = await fixture().run()
+  assert.equal(defaultRun.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(defaultRun.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+
+  for (const failure of ['stream', 'abort'] as const) {
+    const controller = new AbortController()
+    const broken = fixture({ failure, signal: controller.signal, onSecond: () => { if (failure === 'abort') controller.abort() } })
+    const failed = await broken.run(correct)
+    const row = failed.perTask[0]!
+    assert.equal(row.ok, false)
+    assert.equal(row.measurement, 'unavailable')
+    assert.equal(row.prompts?.length, 2)
+    assert.equal(row.events?.length, 4)
+    assert.deepEqual(row.usage, { input: 22, output: 6, costUsd: 0.04, tokensKnown: false, usdKnown: false })
+    assert.equal(row.prompts?.[0]?.usage.tokensKnown, undefined)
+    assert.equal(row.prompts?.[1]?.usage.tokensKnown, false)
+    assert.equal(broken.operations.filter((op) => op === 'delete').length, 1)
+    assert.equal(broken.operations.includes('extract'), false)
+  }
+  const policyFailure = fixture()
+  const policyFailed = await policyFailure.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    throw new Error('policy failed after paid work')
+  })
+  assert.equal(policyFailed.perTask[0]?.ok, false)
+  assert.deepEqual(policyFailed.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+  assert.match(policyFailed.perTask[0]?.detail ?? '', /policy failed/)
+  assert.equal(policyFailure.operations.filter((op) => op === 'delete').length, 1)
+
+  const policyAbortController = new AbortController()
+  const policyAbort = fixture({ signal: policyAbortController.signal })
+  const abortedPolicy = await policyAbort.run(async ({ run, prompt }) => {
+    await run.start(prompt)
+    policyAbortController.abort()
+    await new Promise<void>(() => {})
+  })
+  assert.equal(abortedPolicy.perTask[0]?.ok, false, 'cancellation stops waiting for policy work')
+  assert.equal(abortedPolicy.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(abortedPolicy.perTask[0]?.usage, { input: 11, output: 3, costUsd: 0.02 })
+  assert.equal(policyAbort.operations.filter((op) => op === 'delete').length, 1)
+
+  const looped = fixture()
+  const retried = await looped.run(async ({ run, prompt, attempt }) => {
+    await run.start(prompt)
+    await run.resume(attempt === 2 ? 'repair the missing branch' : 'check again')
+  }, 2)
+  assert.equal(retried.perTask[0]?.resolved, true)
+  assert.equal(looped.creates, 2)
+  assert.deepEqual(retried.perTask[0]?.prompts?.map((p) => [p.attempt, p.index]), [[1, 0], [1, 1], [2, 0], [2, 1]])
+  assert.deepEqual(retried.perTask[0]?.usage, { input: 44, output: 12, costUsd: 0.08 })
+  assert.equal(looped.operations.filter((op) => op === 'setup').length, 2)
+  assert.equal(looped.operations.filter((op) => op === 'extract').length, 2)
+  assert.equal(looped.operations.filter((op) => op === 'delete').length, 2)
+
+  const unawaited = fixture()
+  const awaitedByOwner = await unawaited.run(async ({ run, prompt }) => { void run.start(prompt) })
+  assert.equal(awaitedByOwner.perTask[0]?.prompts?.length, 1)
+  assert.deepEqual(unawaited.operations, ['setup', 'prompt:1', 'extract', 'delete', 'judge'])
+  const overlapping = fixture()
+  const overlap = await overlapping.run(async ({ run, prompt }) => {
+    const first = run.start(prompt)
+    assert.throws(() => run.resume('overlap'), /sequential/)
+    await first
+  })
+  assert.equal(overlap.perTask[0]?.ok, false)
+  assert.equal(overlapping.operations.includes('extract'), false)
+  let retained: BenchExecutionContext['run'] | undefined
+  await fixture().run(async ({ run, prompt }) => { retained = run; await run.start(prompt) })
+  assert.throws(() => retained!.resume('too late'), /settled/)
+  const skipped = await fixture().run(async () => {})
+  assert.equal(skipped.perTask[0]?.ok, false)
+  assert.equal(skipped.perTask[0]?.prompts?.length, 0)
+  const passthrough: BenchExecution = async () => {}
+  let received: BenchExecution | undefined
+  await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'custom', model: 'm' }], n: 1,
+    routerBaseUrl: 'unused', routerKey: 'unused', resolveAdapter: resolveStub,
+    execute: passthrough, runShot: async ({ execute }) => { received = execute; return { artifact: '', ok: false } },
+  })
+  assert.equal(received, passthrough, 'custom shots choose how to consume the callback')
+}
+
 void main()