@tangle-network/agent-bench 0.8.32 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.32",
+  "version": "0.9.1",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.174.0 <0.175.0",
+    "@tangle-network/agent-eval": ">=0.175.0 <0.176.0",
     "@tangle-network/agent-interface": "^2.3.0",
-    "@tangle-network/agent-knowledge": "^14.0.0",
+    "@tangle-network/agent-knowledge": "^14.0.3",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.195.0"
+    "@tangle-network/agent-runtime": "^0.199.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/verify-pier-agent.mts

@@ -37,7 +37,7 @@ import {
   type ResolvedAgentCandidateContainer,
   sealAgentCandidateBundle,
   verifyAgentCandidateBundle,
-} from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
 
 import { executePreparedPierCandidate } from '../src/pier-agent'
 import { createPierResultGrader } from '../src/pier-result-grader'

package/scripts/verify-pier-recovery.mts

@@ -4,7 +4,8 @@ import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import { InMemoryTraceStore } from '@tangle-network/agent-eval'
 
 import { createStagedPierCandidateExecutionFixture } from '../src/pier-agent.test-fixtures.mts'

package/src/index.ts

@@ -65,6 +65,7 @@ export {
   printBenchmarksReport,
   type BenchCell,
   type BenchShot,
+  type BenchShotResult,
   type BenchCellTaskResult,
   type BenchLeaderboardRow,
   type RunBenchmarksOptions,

package/src/official-optimizer-config.mts

@@ -7,8 +7,7 @@ import {
   canonicalAgentProfileDigest,
   canonicalCandidateDigest,
 } from '@tangle-network/agent-interface'
-import { profileOptimizerModelCall } from '../../src/runtime/profile-chat-client'
-import type { RouterSeam } from '../../src/runtime/supervise/runtime'
+import { profileOptimizerModelCall, type RouterSeam } from '@tangle-network/agent-runtime/kernel'
 
 function requiredNonNegativeNumber(
   env: NodeJS.ProcessEnv,
@@ -73,6 +72,7 @@ export function officialOptimizerModel(options: {
   temperature?: number
   reasoningEffort?: NonNullable<AgentProfile['model']>['reasoningEffort']
   callRef?: string
+  anthropicEndpoint?: boolean
   complete?: RouterSeam['complete']
 }): OpenAICompatibleOptimizerModel {
   const { env } = options
@@ -106,12 +106,20 @@ export function officialOptimizerModel(options: {
     routerKey: options.apiKey,
     ...(options.complete ? { complete: options.complete } : {}),
   }
-  const call = profileOptimizerModelCall({
-    profile,
-    context: 'official optimizer model',
-    executor,
-    pricing: budget.pricing,
-  })
+  const call: OpenAICompatibleOptimizerModel['call'] = (request) =>
+    profileOptimizerModelCall({
+      // Eval admits the request against the budget; execute its exact output limit.
+      profile: {
+        ...profile,
+        model: {
+          ...profile.model,
+          maxVisibleOutputTokens: request.request.maxTokens ?? options.maxOutputTokensPerRequest,
+        },
+      },
+      context: 'official optimizer model',
+      executor,
+      pricing: budget.pricing,
+    })(request)
   return {
     model: options.model,
     callRef:
@@ -122,6 +130,7 @@ export function officialOptimizerModel(options: {
       })}`,
     call,
     budget,
+    ...(options.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}),
   }
 }
 

package/src/pier-agent.test.mts

@@ -9,7 +9,7 @@ import { canonicalJson } from '@tangle-network/agent-eval'
 import type {
   AgentCandidateExecutorRequest,
   PreparedAgentCandidateExecution,
-} from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
 
 import {
   awaitAbortableTrial,

package/src/pier-agent.ts

@@ -14,8 +14,9 @@ import type {
   AgentCandidateProtectedRunCapture,
   AgentCandidateRunFinalization,
   PreparedAgentCandidateExecution,
-} from '@tangle-network/agent-runtime'
-import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
+
+import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime/candidate-execution'
 import { canonicalJson, type TraceStore } from '@tangle-network/agent-eval'
 
 import { capturePierTaskOutcome } from './pier-task-outcome'

package/src/pier-trial-controller.test.mts

@@ -6,12 +6,14 @@ import path from 'node:path'
 import test from 'node:test'
 import { promisify } from 'node:util'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import { InMemoryTraceStore } from '@tangle-network/agent-eval'
 
 import { createStagedPierCandidateExecutionFixture } from './pier-agent.test-fixtures.mts'
 import { FilePierCandidateTrialController } from './pier-trial-controller'
 
+// Process identity and restart recovery require Linux /proc start-time receipts.
 const execFileAsync = promisify(execFile)
 
 function testRequest(executionId: string, executionPlanDigest: `sha256:${string}`) {
@@ -71,7 +73,7 @@ test('an existing Pier job is rejected without deleting or starting it', async (
   }
 })
 
-test('the Pier result wait does not add time beyond an expired deadline', async () => {
+test('the Pier result wait does not add time beyond an expired deadline', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-deadline-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -144,7 +146,7 @@ setInterval(() => undefined, 1_000)
   }
 })
 
-test('the supervisor receives only launch data and no inherited evaluator environment', async () => {
+test('the supervisor receives only launch data and no inherited evaluator environment', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-supervisor-env-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -217,7 +219,7 @@ renameSync(temporary, target)
   }
 })
 
-test('terminal acknowledgements reject unknown fields', async () => {
+test('terminal acknowledgements reject unknown fields', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-terminal-schema-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -274,7 +276,7 @@ renameSync(temporary, target)
   }
 })
 
-test('a fresh evaluator process terminates the persisted process and container identity', async () => {
+test('a fresh evaluator process terminates the persisted process and container identity', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-recovery-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')

package/src/pier-trial-controller.ts

@@ -21,7 +21,8 @@ import { isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { Writable } from 'node:stream'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import type { TraceStore } from '@tangle-network/agent-eval'
 
 import type {

package/src/run-benchmarks.test.mts

@@ -60,6 +60,41 @@ async function main(): Promise<void> {
   assert.equal(report.perTask.length, 24, 'matrix expands to benchmarks × cells × tasks')
   assert.equal(report.rows.length, 6, 'one row per (benchmark × cell)')
 
+  const measured = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'measured', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1,
+    runShot: async () => ({ artifact: 'WRONG', ok: true, usage: { input: 23, output: 7, costUsd: 0.04 } }),
+  })
+  assert.equal(measured.perTask[0]?.artifact, 'WRONG', 'the exact judged artifact reaches the caller even when it fails')
+  assert.deepEqual(measured.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 }, 'measured usage reaches the caller')
+
+  const judgeFailure = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'measured', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', n: 1, verifyJudge: false,
+    resolveAdapter: () => ({ ...REGISTRY.alpha!, judge: async () => { throw new Error('judge unavailable') } }),
+    runShot: async () => ({ artifact: 'PATCH', ok: true, usage: { input: 23, output: 7, costUsd: 0.04 } }),
+  })
+  assert.equal(judgeFailure.perTask[0]?.ok, false)
+  assert.equal(judgeFailure.perTask[0]?.artifact, 'PATCH', 'a judge outage retains the completed agent artifact')
+  assert.equal(judgeFailure.perTask[0]?.usage?.costUsd, 0.04, 'a judge outage retains already incurred usage')
+
+  const controller = new AbortController()
+  let started = 0
+  const cancelled = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'cancelled', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, concurrency: 1,
+    signal: controller.signal,
+    runShot: async ({ signal }) => {
+      assert.equal(signal, controller.signal)
+      started += 1
+      controller.abort()
+      return { artifact: 'PATCH', ok: true, usage: { input: 1, output: 2, costUsd: 0.01 } }
+    },
+  })
+  assert.equal(started, 1, 'cancellation prevents every queued model call')
+  assert.equal(cancelled.perTask.length, 4, 'cancelled work remains visible')
+  assert.equal(cancelled.perTask[0]?.usage?.costUsd, 0.01)
+
   const row = (b: string, c: string) => report.rows.find((r) => r.benchmark === b && r.cell === c)!
   assert.equal(row('alpha', 'perfect').resolveRate, 1, 'perfect cell resolves every task')
   assert.equal(row('alpha', 'half').resolveRate, 0.5, 'half cell resolves the even tasks')
@@ -94,7 +129,7 @@ async function main(): Promise<void> {
   let prompts: string[] = []
   const retryShot: BenchShot = async ({ task, prompt }) => {
     prompts.push(prompt ?? task.prompt)
-    return { artifact: prompts.length === 1 ? 'WRONG' : String(task.metadata?.gold), ok: true }
+    return { artifact: prompts.length === 1 ? 'WRONG' : String(task.metadata?.gold), ok: true, usage: { input: 11, output: 3, costUsd: 0.02 } }
   }
   const oneShot = await runBenchmarks({
     benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
@@ -110,6 +145,40 @@ async function main(): Promise<void> {
   assert.equal(prompts.length, 2, 'loop stops after the passing second attempt')
   assert.match(prompts[1]!, /Previous attempts and safe checker feedback/)
   assert.match(looped.perTask[0]!.detail ?? '', /"mode":"refine-loop"/)
+  assert.deepEqual(looped.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 }, 'retry cost includes the rejected attempt')
+
+  const failedRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ attempt }) => {
+      if (attempt === 2) throw new Error('connection lost after dispatch')
+      return { artifact: 'WRONG', ok: true, usage: { input: 11, output: 3, costUsd: 0.02 } }
+    },
+  })
+  assert.deepEqual(failedRetry.perTask[0]?.usage, {
+    input: 11, output: 3, costUsd: 0.02, tokensKnown: false, usdKnown: false,
+  }, 'an unreported retry preserves the measured floor without claiming complete accounting')
+
+  let validAttempts = 0
+  const validRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 3,
+    runShot: async ({ task, attempt }) => {
+      validAttempts += 1
+      return { artifact: attempt === 1 ? 'WRONG' : String(task.metadata?.gold), ok: attempt !== 2 }
+    },
+  })
+  assert.equal(validAttempts, 3, 'a failed execution cannot stop refinement with apparent gold')
+  assert.equal(validRetry.perTask[0]?.ok, true)
+  const validIncumbent = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ task, attempt }) => ({
+      artifact: attempt === 1 ? 'WRONG' : String(task.metadata?.gold), ok: attempt === 1,
+    }),
+  })
+  assert.equal(validIncumbent.perTask[0]?.artifact, 'WRONG', 'a failed execution cannot replace a valid incumbent')
+  assert.equal(validIncumbent.perTask[0]?.ok, true)
 
   // A benchmark's detail may include hidden answer fields; those must never be fed back as hints.
   const leakyGold = 'SECRET-GOLD'
@@ -141,8 +210,11 @@ async function main(): Promise<void> {
   if (runtime.openSandboxRun.toString().includes('beforeStart')) {
     // The default shot path supports benchmark-owned box setup/extract without real sandbox infra.
     const order: string[] = []
+    let createdOptions: unknown
+    let controlCredential: string | undefined
     const fakeClient = {
-      async create() {
+      async create(options: unknown) {
+        createdOptions = options
         return {
           id: 'box-default-shot',
           async exec(command: string, options?: { sessionId?: string }) {
@@ -151,7 +223,9 @@ async function main(): Promise<void> {
           },
           async *streamPrompt(_prompt: string, options?: { sessionId?: string }) {
             order.push(`stream:session=${options?.sessionId ? 'yes' : 'no'}`)
+            yield { type: 'llm_call', data: { tokensIn: 23, tokensOut: 7, costUsd: 0.04 } }
             yield { type: 'result', data: { finalText: 'fallback text' } }
+            yield { type: 'done', data: { outcome: { type: 'completed' } } }
           },
           async delete() {
             order.push('delete')
@@ -175,11 +249,19 @@ async function main(): Promise<void> {
       benchmarks: ['boxy'],
       cells: [{ label: 'default-shot', model: 'm', backend: 'sandbox' }],
       routerBaseUrl: 'x',
-      routerKey: 'x',
+      routerKey: 'sandbox-control-token',
+      modelApiKey: 'model-grant-token',
       resolveAdapter: () => boxAdapter,
-      resolveClient: () => fakeClient as never,
+      resolveClient: (options) => {
+        controlCredential = options.routerKey
+        return fakeClient as never
+      },
     })
     assert.equal(boxy.rows[0]!.resolveRate, 1, 'boxExtract artifact is judged instead of fallback text')
+    assert.equal(boxy.perTask[0]?.artifact, 'PATCH')
+    assert.deepEqual(boxy.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 })
+    assert.equal(controlCredential, 'sandbox-control-token', 'inference grant never authorizes sandbox control')
+    assert.equal((createdOptions as { backend: { model: { apiKey: string } } }).backend.model.apiKey, 'model-grant-token')
     assert.deepEqual(
       order.slice(0, 3),
       ['exec:setup-repo:streams=0:session=yes', 'stream:session=yes', 'exec:extract-patch:streams=1:session=yes'],

package/src/run-benchmarks.ts

@@ -35,7 +35,7 @@ import type {
   Deliverable,
   OpenSandboxRunOptions,
 } from '@tangle-network/agent-runtime/kernel'
-import { openSandboxRun } from '@tangle-network/agent-runtime/kernel'
+import { openSandboxRun, SandboxRunAbortError, sumSandboxUsage } from '@tangle-network/agent-runtime/kernel'
 import type { SandboxEvent } from '@tangle-network/sandbox'
 import { resolveAdapter } from './adapters'
 import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
@@ -60,8 +60,17 @@ export interface BenchCell {
   readonly profile?: AgentProfile
 }
 
-/** Runs one (adapter, task, cell) shot and returns the deliverable text. The default uses
- *  `openSandboxRun`; tests inject a deterministic stub so the matrix runs offline. */
+/** A worker's artifact and observed execution evidence, before external grading. */
+export interface BenchShotResult {
+  readonly artifact: string
+  readonly ok: boolean
+  readonly detail?: string
+  /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
+  readonly usage?: ReturnType<typeof sumSandboxUsage>
+  readonly events?: readonly SandboxEvent[]
+}
+
+/** Runs one (adapter, task, cell) shot. Defaults to `openSandboxRun`. */
 export type BenchShot = (input: {
   readonly adapter: BenchmarkAdapter
   readonly task: BenchTask
@@ -72,12 +81,15 @@ export type BenchShot = (input: {
   readonly attempt?: number
   readonly routerBaseUrl: string
   readonly routerKey: string
+  /** Optional inference credential for the box; routerKey continues to authorize sandbox control. */
+  readonly modelApiKey?: string
   readonly bridgeUrl?: string
   readonly bridgeBearer?: string
   readonly sandboxBaseUrl?: string
   readonly timeoutMs?: number
+  readonly signal?: AbortSignal
   readonly resolveClient?: typeof resolveBenchClient
-}) => Promise<{ artifact: string; ok: boolean; detail?: string }>
+}) => Promise<BenchShotResult>
 
 export interface RunBenchmarksOptions {
   /** Registry keys (`resolveAdapter`) — the benchmark subset to run. */
@@ -86,6 +98,8 @@ export interface RunBenchmarksOptions {
   readonly cells: readonly BenchCell[]
   readonly routerBaseUrl: string
   readonly routerKey: string
+  /** Optional inference credential for the box; never used for sandbox creation or deletion. */
+  readonly modelApiKey?: string
   readonly bridgeUrl?: string
   readonly bridgeBearer?: string
   readonly sandboxBaseUrl?: string
@@ -99,6 +113,8 @@ export interface RunBenchmarksOptions {
   readonly concurrency?: number
   /** Per-shot wall-clock (ms). */
   readonly timeoutMs?: number
+  /** Cancels active shots and prevents queued shots from starting. */
+  readonly signal?: AbortSignal
   /** Test seam: resolve the runtime transport. Defaults to `resolveBenchClient`. */
   readonly resolveClient?: typeof resolveBenchClient
   /** Max attempts per (benchmark × cell × task). Default 1. Attempts after the first receive
@@ -126,6 +142,11 @@ export interface BenchCellTaskResult {
   readonly ok: boolean
   readonly detail?: string
   readonly wallMs: number
+  /** Exact bytes given to the benchmark judge, retained even when judging fails. */
+  readonly artifact?: string
+  readonly usage?: ReturnType<typeof sumSandboxUsage>
+  /** Worker events only; benchmark grading remains outside this trace. */
+  readonly events?: readonly SandboxEvent[]
 }
 
 export interface BenchLeaderboardRow {
@@ -173,7 +194,8 @@ function finalText(events: readonly SandboxEvent[]): string {
 
 /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable
  *  extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */
-const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, resolveClient }) => {
+const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
+  signal?.throwIfAborted()
   const client = (resolveClient ?? resolveBenchClient)({
     backend: cell.backend ?? 'router',
     routerBaseUrl,
@@ -206,7 +228,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     sandboxOverrides: {
       name: `bench-${adapter.name}-${task.id}-${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl } },
+      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl, ...(modelApiKey === undefined ? {} : { apiKey: modelApiKey }) } },
     },
   }
   const deliverable: Deliverable<string> = {
@@ -217,7 +239,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : undefined
   const runOptions: OpenSandboxRunOptions = {
     agentRun,
-    signal: controller.signal,
+    signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
     runId: `bench:${adapter.name}:${task.id}:${uniq}`,
     scenarioId: task.id,
   }
@@ -236,9 +258,12 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
         )
     }
   }
-  const run = await openSandboxRun(client, runOptions, deliverable)
+  let run: Awaited<ReturnType<typeof openSandboxRun<string>>> | undefined
+  let result: BenchShotResult = { artifact: '', ok: false }
   try {
+    run = await openSandboxRun(client, runOptions, deliverable)
     const turn = await run.start(prompt ?? task.prompt)
+    result = { artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events }
     // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK.
     let artifact = (turn.out ?? '').trim()
     let boxExtractError: string | undefined
@@ -289,7 +314,9 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
       }
     }
     const detail =
-      turn.readError !== undefined
+      !turn.outcome.success
+        ? turn.outcome.error ?? `agent ended with status ${turn.outcome.status}`
+        : turn.readError !== undefined
         ? `read: ${turn.readError.slice(0, 160)}`
         : boxExtractError !== undefined
           ? `boxExtract: ${boxExtractError}`
@@ -306,15 +333,30 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
         // debug-only; never fail the shot on a dump error
       }
     }
-    return {
+    result = {
       artifact,
-      ok: artifact.length > 0,
+      ok: turn.outcome.success && artifact.length > 0 && turn.readError === undefined && boxExtractError === undefined,
+      usage: result.usage,
+      events: turn.events,
       ...(detail ? { detail } : {}),
     }
+  } catch (err) {
+    result = {
+      ...result,
+      ok: false,
+      detail: err instanceof Error ? err.message : String(err),
+      ...(err instanceof SandboxRunAbortError ? { usage: sumSandboxUsage(err.events), events: err.events } : {}),
+    }
   } finally {
     if (timer) clearTimeout(timer)
-    await run.close()
+    try {
+      await run?.close()
+    } catch (err) {
+      const cleanup = `cleanup: ${err instanceof Error ? err.message : String(err)}`
+      result = { ...result, detail: combineDetails(result.detail, cleanup) }
+    }
   }
+  return result
 }
 
 function parseMaybeJson(value: string): unknown {
@@ -380,23 +422,44 @@ async function loopedShot(
   input: Parameters<BenchShot>[0],
   shot: BenchShot,
   attempts: number,
-): Promise<{ artifact: string; ok: boolean; detail?: string }> {
+): Promise<BenchShotResult> {
   const scores = new Map<number, BenchScore>()
-  const result = await runRefineLoop<string>({
-    rounds: attempts,
-    prompt: (round, history) => (round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores)),
-    runShot: async (prompt, round) => {
-      const out = await shot({ ...input, prompt, attempt: round })
-      return { artifact: out.artifact, note: out.detail }
-    },
-    judge: async (artifact, round) => {
-      const score = await input.adapter.judge(input.task, artifact)
-      scores.set(round, score)
-      return { valid: score.resolved, score: score.score }
-    },
-  })
+  const shots = new Map<number, BenchShotResult>()
+  let pendingShot = false
+  let result: Awaited<ReturnType<typeof runRefineLoop<string>>>
+  try {
+    result = await runRefineLoop<string>({
+      rounds: attempts,
+      prompt: (round, history) => (round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores)),
+      runShot: async (prompt, round) => {
+        input.signal?.throwIfAborted()
+        pendingShot = true
+        const out = await shot({ ...input, prompt, attempt: round })
+        shots.set(round, out)
+        pendingShot = false
+        return { artifact: out.artifact, note: out.detail }
+      },
+      judge: async (artifact, round) => {
+        const score = await input.adapter.judge(input.task, artifact)
+        scores.set(round, score)
+        const succeeded = shots.get(round)?.ok === true
+        return { valid: succeeded && score.resolved, score: succeeded ? score.score : 0 }
+      },
+    })
+  } catch (err) {
+    const completed = [...shots.values()]
+    return {
+      artifact: completed.at(-1)?.artifact ?? '',
+      ok: false,
+      usage: combinedUsage(pendingShot ? [...completed, { artifact: '', ok: false }] : completed),
+      events: completed.flatMap((shot) => shot.events ?? []),
+      detail: err instanceof Error ? err.message : String(err),
+    }
+  }
 
   const best = result.rounds.reduce((winner, candidate) => {
+    if (shots.get(candidate.round)?.ok !== true) return winner
+    if (shots.get(winner.round)?.ok !== true) return candidate
     const a = scores.get(winner.round)
     const b = scores.get(candidate.round)
     if (!a) return candidate
@@ -408,7 +471,9 @@ async function loopedShot(
   const bestScore = scores.get(best.round)
   return {
     artifact: best.artifact,
-    ok: best.artifact.trim().length > 0,
+    ok: shots.get(best.round)?.ok === true && best.artifact.trim().length > 0,
+    usage: combinedUsage([...shots.values()]),
+    events: [...shots.values()].flatMap((shot) => shot.events ?? []),
     detail: JSON.stringify({
       mode: 'refine-loop',
       attempts: result.rounds.length,
@@ -425,6 +490,30 @@ async function loopedShot(
   }
 }
 
+function combinedUsage(shots: readonly BenchShotResult[]): ReturnType<typeof sumSandboxUsage> {
+  const usage = { input: 0, output: 0, costUsd: 0 }
+  let tokensKnown = shots.length > 0
+  let usdKnown = shots.length > 0
+  let estimate: number | undefined
+  let unknownReason: string | undefined
+  for (const shot of shots) {
+    usage.input += shot.usage?.input ?? 0
+    usage.output += shot.usage?.output ?? 0
+    usage.costUsd += shot.usage?.costUsd ?? 0
+    tokensKnown &&= shot.usage !== undefined && shot.usage.tokensKnown !== false
+    usdKnown &&= shot.usage !== undefined && shot.usage.usdKnown !== false
+    unknownReason ??= shot.usage?.tokensUnknownReason
+    if (shot.usage?.estimatedCostUsd !== undefined) estimate = (estimate ?? 0) + shot.usage.estimatedCostUsd
+  }
+  return {
+    ...usage,
+    ...(tokensKnown ? {} : { tokensKnown: false as const }),
+    ...(usdKnown ? {} : { usdKnown: false as const }),
+    ...(estimate === undefined ? {} : { estimatedCostUsd: estimate }),
+    ...(unknownReason === undefined ? {} : { tokensUnknownReason: unknownReason }),
+  }
+}
+
 function combineDetails(runDetail: string | undefined, scoreDetail: string | undefined): string | undefined {
   if (runDetail && scoreDetail) {
     return JSON.stringify({ run: parseMaybeJson(runDetail), score: parseMaybeJson(scoreDetail) })
@@ -481,6 +570,7 @@ async function prepareBenchmarks(
 }
 
 export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenchmarksReport> {
+  opts.signal?.throwIfAborted()
   if (opts.benchmarks.length === 0) throw new Error('runBenchmarks: no benchmarks selected')
   if (opts.cells.length === 0) throw new Error('runBenchmarks: no cells to run')
   const reps = Math.max(1, opts.reps ?? 1)
@@ -497,20 +587,24 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
   await runPool(jobs, Math.max(1, opts.concurrency ?? 4), async (job, index) => {
     const startedAt = Date.now()
     let result: BenchCellTaskResult
+    let out: BenchShotResult | undefined
     try {
+      opts.signal?.throwIfAborted()
       const shotInput = {
         adapter: job.adapter,
         task: job.task,
         cell: job.cell,
         routerBaseUrl: opts.routerBaseUrl,
         routerKey: opts.routerKey,
+        ...(opts.modelApiKey === undefined ? {} : { modelApiKey: opts.modelApiKey }),
         ...(opts.bridgeUrl ? { bridgeUrl: opts.bridgeUrl } : {}),
         ...(opts.bridgeBearer ? { bridgeBearer: opts.bridgeBearer } : {}),
         ...(opts.sandboxBaseUrl ? { sandboxBaseUrl: opts.sandboxBaseUrl } : {}),
         ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}),
+        ...(opts.signal ? { signal: opts.signal } : {}),
         ...(opts.resolveClient ? { resolveClient: opts.resolveClient } : {}),
       }
-      const out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
+      out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
       const score: BenchScore = await job.adapter.judge(job.task, out.artifact)
       result = {
         benchmark: job.benchmark,
@@ -522,6 +616,9 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ok: out.ok,
         ...(out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {}),
         wallMs: Date.now() - startedAt,
+        artifact: out.artifact,
+        ...(out.usage === undefined ? {} : { usage: out.usage }),
+        ...(out.events === undefined ? {} : { events: out.events }),
       }
     } catch (err) {
       // A thrown shot/judge is infra error for THIS cell-task: ok=false excludes it from the
@@ -536,6 +633,9 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ok: false,
         detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
         wallMs: Date.now() - startedAt,
+        ...(out === undefined ? {} : { artifact: out.artifact }),
+        ...(out?.usage === undefined ? {} : { usage: out.usage }),
+        ...(out?.events === undefined ? {} : { events: out.events }),
       }
     }
     void index

package/src/sandbox-run.ts

@@ -14,7 +14,8 @@ import {
   type AgentRunSpec,
   type OutputAdapter,
 } from '@tangle-network/agent-runtime/kernel'
-import { parseExactAgentProfile } from '@tangle-network/agent-runtime'
+import { parseExactAgentProfile } from '@tangle-network/agent-runtime/candidate-execution'
+
 // `BackendType` is the sandbox SDK's harness union and its canonical home. Runtime consumes it
 // from there too; benchmark profiles use the same values as their exact harness identity.
 import type { BackendType } from '@tangle-network/sandbox'

package/src/swe-arena/activation.test.mts

@@ -211,6 +211,7 @@ describe('activation-predicate prefilter', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'act-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'act-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
     await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
     await writeFile(join(loopsRepo, 'src.ts'), 'base\n')