@tangle-network/agent-bench 0.8.23 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.23",
+  "version": "0.9.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,18 +25,18 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.163.2 <0.164.0",
-    "@tangle-network/agent-interface": "^1.6.0",
-    "@tangle-network/agent-knowledge": "^10.7.0",
-    "@tangle-network/sandbox": ">=0.31.0 <0.32.0",
-    "@tangle-network/agent-runtime": "^0.157.0"
+    "@tangle-network/agent-eval": ">=0.175.0 <0.176.0",
+    "@tangle-network/agent-interface": "^2.3.0",
+    "@tangle-network/agent-knowledge": "^14.0.3",
+    "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
+    "@tangle-network/agent-runtime": "^0.198.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",
-    "@types/node": "26.1.1",
-    "publint": "0.3.22",
+    "@types/node": "26.4.0",
+    "publint": "0.3.24",
     "tsdown": "0.22.14",
-    "tsx": "4.23.1",
+    "tsx": "4.23.12",
     "typescript": "6.0.3"
   },
   "engines": {

package/scripts/verify-pier-agent.mts

@@ -37,7 +37,7 @@ import {
   type ResolvedAgentCandidateContainer,
   sealAgentCandidateBundle,
   verifyAgentCandidateBundle,
-} from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
 
 import { executePreparedPierCandidate } from '../src/pier-agent'
 import { createPierResultGrader } from '../src/pier-result-grader'

package/scripts/verify-pier-recovery.mts

@@ -4,7 +4,8 @@ import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import { InMemoryTraceStore } from '@tangle-network/agent-eval'
 
 import { createStagedPierCandidateExecutionFixture } from '../src/pier-agent.test-fixtures.mts'

package/src/atom-mcp-e2e.mts

@@ -176,6 +176,7 @@ async function main(): Promise<void> {
         blobs,
         makeWorkerAgent: (raw) => makeWorker(raw, ws, n++),
         perWorker: { maxIterations: 2, maxTokens: 200_000 },
+        toolNames: ['spawn_worker', 'await_event', 'stop'],
       })
       // The supervisor's cwd carries the REAL skill file (opencode loads it from the cwd skill dirs).
       const supCwd = mkdtempSync(join(tmpdir(), 'e2e-sup-'))

package/src/benchmarks/humaneval.test.mts

@@ -13,8 +13,8 @@ describe('HumanEval Python isolation', () => {
     writeFileSync(
       fakeDocker,
       `#!/usr/bin/env node
-import fs from 'node:fs'
-import path from 'node:path'
+const fs = require('node:fs')
+const path = require('node:path')
 const args = process.argv.slice(2)
 if (args[0] === 'rm') process.exit(0)
 if (process.env.FAKE_DOCKER_MISSING === '1') {

package/src/coordination-mcp-container-reach.mts

@@ -118,6 +118,7 @@ async function main(): Promise<void> {
         blobs,
         makeWorkerAgent: () => trivialWorker('w'),
         perWorker: { maxIterations: 4, maxTokens: 2000 },
+        toolNames: ['spawn_worker', 'await_event'],
         host: HOST_BIND,
       })
       // Docker containers reach the host through the bridge gateway, not the 0.0.0.0 bind URL.
@@ -139,10 +140,10 @@ async function main(): Promise<void> {
         console.error('')
 
         const toolNames = (parsed.result?.tools ?? []).map((t) => t.name)
-        const hasSpawn = toolNames.includes('spawn_agent')
+        const hasSpawn = toolNames.includes('spawn_worker')
         const hasAwait = toolNames.includes('await_event')
         console.error(`[probe] tools advertised: ${toolNames.join(', ')}`)
-        console.error(`[probe] spawn_agent present = ${hasSpawn}; await_event present = ${hasAwait}`)
+        console.error(`[probe] spawn_worker present = ${hasSpawn}; await_event present = ${hasAwait}`)
         ok = hasSpawn && hasAwait
 
         if (ok) {
@@ -169,7 +170,7 @@ async function main(): Promise<void> {
 
   console.error(
     ok
-      ? 'CONTAINER-REACHABLE: docker tools/list returned spawn_agent and await_event.'
+      ? 'CONTAINER-REACHABLE: docker tools/list returned spawn_worker and await_event.'
       : 'NOT reachable from container; see output above.',
   )
   process.exit(ok ? 0 : 1)

package/src/index.ts

@@ -65,6 +65,7 @@ export {
   printBenchmarksReport,
   type BenchCell,
   type BenchShot,
+  type BenchShotResult,
   type BenchCellTaskResult,
   type BenchLeaderboardRow,
   type RunBenchmarksOptions,

package/src/mcp-mount-probe.mts

@@ -90,6 +90,7 @@ async function main(): Promise<void> {
         blobs,
         makeWorkerAgent: () => deliveringLeaf('w', { ok: true }),
         perWorker: { maxIterations: 4, maxTokens: 2000 },
+        toolNames: ['spawn_worker', 'await_event', 'stop'],
       })
       console.error(`[probe] coordination MCP live at ${mcp.url}`)
       try {

package/src/pier-agent.test.mts

@@ -9,7 +9,7 @@ import { canonicalJson } from '@tangle-network/agent-eval'
 import type {
   AgentCandidateExecutorRequest,
   PreparedAgentCandidateExecution,
-} from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
 
 import {
   awaitAbortableTrial,

package/src/pier-agent.ts

@@ -14,8 +14,9 @@ import type {
   AgentCandidateProtectedRunCapture,
   AgentCandidateRunFinalization,
   PreparedAgentCandidateExecution,
-} from '@tangle-network/agent-runtime'
-import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime'
+} from '@tangle-network/agent-runtime/candidate-execution'
+
+import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime/candidate-execution'
 import { canonicalJson, type TraceStore } from '@tangle-network/agent-eval'
 
 import { capturePierTaskOutcome } from './pier-task-outcome'
@@ -219,8 +220,12 @@ async function assertRealDirectory(path: string, label: string): Promise<void> {
   if (!stats.isDirectory() || stats.isSymbolicLink()) {
     throw new Error(`${label} must be a real directory`)
   }
-  if ((await realpath(path)) !== path) {
-    throw new Error(`${label} has a symlinked path component`)
+  // A symlinked PREFIX is the OS's own doing on macOS, where a temp root lives under /var, a
+  // symlink to /private/var. The directory itself must still be real, which the lstat above
+  // and this one both require.
+  const resolvedStats = await lstat(await realpath(path))
+  if (!resolvedStats.isDirectory() || resolvedStats.isSymbolicLink()) {
+    throw new Error(`${label} must resolve to a real directory`)
   }
 }
 

package/src/pier-trial-controller.test.mts

@@ -6,12 +6,14 @@ import path from 'node:path'
 import test from 'node:test'
 import { promisify } from 'node:util'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import { InMemoryTraceStore } from '@tangle-network/agent-eval'
 
 import { createStagedPierCandidateExecutionFixture } from './pier-agent.test-fixtures.mts'
 import { FilePierCandidateTrialController } from './pier-trial-controller'
 
+// Process identity and restart recovery require Linux /proc start-time receipts.
 const execFileAsync = promisify(execFile)
 
 function testRequest(executionId: string, executionPlanDigest: `sha256:${string}`) {
@@ -71,7 +73,7 @@ test('an existing Pier job is rejected without deleting or starting it', async (
   }
 })
 
-test('the Pier result wait does not add time beyond an expired deadline', async () => {
+test('the Pier result wait does not add time beyond an expired deadline', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-deadline-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -144,7 +146,7 @@ setInterval(() => undefined, 1_000)
   }
 })
 
-test('the supervisor receives only launch data and no inherited evaluator environment', async () => {
+test('the supervisor receives only launch data and no inherited evaluator environment', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-supervisor-env-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -217,7 +219,7 @@ renameSync(temporary, target)
   }
 })
 
-test('terminal acknowledgements reject unknown fields', async () => {
+test('terminal acknowledgements reject unknown fields', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-terminal-schema-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')
@@ -274,7 +276,7 @@ renameSync(temporary, target)
   }
 })
 
-test('a fresh evaluator process terminates the persisted process and container identity', async () => {
+test('a fresh evaluator process terminates the persisted process and container identity', { skip: process.platform !== 'linux' }, async () => {
   const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-recovery-'))
   const controlRoot = path.join(root, 'control')
   const jobsDirectory = path.join(root, 'jobs')

package/src/pier-trial-controller.ts

@@ -21,7 +21,8 @@ import { isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { Writable } from 'node:stream'
 
-import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime/candidate-execution'
+
 import type { TraceStore } from '@tangle-network/agent-eval'
 
 import type {
@@ -493,9 +494,16 @@ function trialKey(identity: PierCandidateTrialIdentity): string {
 
 function assertRealDirectory(path: string, label: string): void {
   const stats = lstatSync(path)
-  if (!stats.isDirectory() || stats.isSymbolicLink() || realpathSync(path) !== path) {
+  if (!stats.isDirectory() || stats.isSymbolicLink()) {
     throw new Error(`${label} must be a real canonical directory`)
   }
+  // A symlinked PREFIX is the OS's own doing on macOS, where a temp root lives under /var, a
+  // symlink to /private/var. The directory itself must still be real, which both this check
+  // and the lstat above require.
+  const resolvedStats = lstatSync(realpathSync(path))
+  if (!resolvedStats.isDirectory() || resolvedStats.isSymbolicLink()) {
+    throw new Error(`${label} must resolve to a real directory`)
+  }
 }
 
 export class FilePierCandidateTrialController implements PierCandidateTrialController {

package/src/run-benchmarks.test.mts

@@ -60,6 +60,41 @@ async function main(): Promise<void> {
   assert.equal(report.perTask.length, 24, 'matrix expands to benchmarks × cells × tasks')
   assert.equal(report.rows.length, 6, 'one row per (benchmark × cell)')
 
+  const measured = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'measured', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1,
+    runShot: async () => ({ artifact: 'WRONG', ok: true, usage: { input: 23, output: 7, costUsd: 0.04 } }),
+  })
+  assert.equal(measured.perTask[0]?.artifact, 'WRONG', 'the exact judged artifact reaches the caller even when it fails')
+  assert.deepEqual(measured.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 }, 'measured usage reaches the caller')
+
+  const judgeFailure = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'measured', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', n: 1, verifyJudge: false,
+    resolveAdapter: () => ({ ...REGISTRY.alpha!, judge: async () => { throw new Error('judge unavailable') } }),
+    runShot: async () => ({ artifact: 'PATCH', ok: true, usage: { input: 23, output: 7, costUsd: 0.04 } }),
+  })
+  assert.equal(judgeFailure.perTask[0]?.ok, false)
+  assert.equal(judgeFailure.perTask[0]?.artifact, 'PATCH', 'a judge outage retains the completed agent artifact')
+  assert.equal(judgeFailure.perTask[0]?.usage?.costUsd, 0.04, 'a judge outage retains already incurred usage')
+
+  const controller = new AbortController()
+  let started = 0
+  const cancelled = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'cancelled', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, concurrency: 1,
+    signal: controller.signal,
+    runShot: async ({ signal }) => {
+      assert.equal(signal, controller.signal)
+      started += 1
+      controller.abort()
+      return { artifact: 'PATCH', ok: true, usage: { input: 1, output: 2, costUsd: 0.01 } }
+    },
+  })
+  assert.equal(started, 1, 'cancellation prevents every queued model call')
+  assert.equal(cancelled.perTask.length, 4, 'cancelled work remains visible')
+  assert.equal(cancelled.perTask[0]?.usage?.costUsd, 0.01)
+
   const row = (b: string, c: string) => report.rows.find((r) => r.benchmark === b && r.cell === c)!
   assert.equal(row('alpha', 'perfect').resolveRate, 1, 'perfect cell resolves every task')
   assert.equal(row('alpha', 'half').resolveRate, 0.5, 'half cell resolves the even tasks')
@@ -94,7 +129,7 @@ async function main(): Promise<void> {
   let prompts: string[] = []
   const retryShot: BenchShot = async ({ task, prompt }) => {
     prompts.push(prompt ?? task.prompt)
-    return { artifact: prompts.length === 1 ? 'WRONG' : String(task.metadata?.gold), ok: true }
+    return { artifact: prompts.length === 1 ? 'WRONG' : String(task.metadata?.gold), ok: true, usage: { input: 11, output: 3, costUsd: 0.02 } }
   }
   const oneShot = await runBenchmarks({
     benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
@@ -110,6 +145,40 @@ async function main(): Promise<void> {
   assert.equal(prompts.length, 2, 'loop stops after the passing second attempt')
   assert.match(prompts[1]!, /Previous attempts and safe checker feedback/)
   assert.match(looped.perTask[0]!.detail ?? '', /"mode":"refine-loop"/)
+  assert.deepEqual(looped.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 }, 'retry cost includes the rejected attempt')
+
+  const failedRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ attempt }) => {
+      if (attempt === 2) throw new Error('connection lost after dispatch')
+      return { artifact: 'WRONG', ok: true, usage: { input: 11, output: 3, costUsd: 0.02 } }
+    },
+  })
+  assert.deepEqual(failedRetry.perTask[0]?.usage, {
+    input: 11, output: 3, costUsd: 0.02, tokensKnown: false, usdKnown: false,
+  }, 'an unreported retry preserves the measured floor without claiming complete accounting')
+
+  let validAttempts = 0
+  const validRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 3,
+    runShot: async ({ task, attempt }) => {
+      validAttempts += 1
+      return { artifact: attempt === 1 ? 'WRONG' : String(task.metadata?.gold), ok: attempt !== 2 }
+    },
+  })
+  assert.equal(validAttempts, 3, 'a failed execution cannot stop refinement with apparent gold')
+  assert.equal(validRetry.perTask[0]?.ok, true)
+  const validIncumbent = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ task, attempt }) => ({
+      artifact: attempt === 1 ? 'WRONG' : String(task.metadata?.gold), ok: attempt === 1,
+    }),
+  })
+  assert.equal(validIncumbent.perTask[0]?.artifact, 'WRONG', 'a failed execution cannot replace a valid incumbent')
+  assert.equal(validIncumbent.perTask[0]?.ok, true)
 
   // A benchmark's detail may include hidden answer fields; those must never be fed back as hints.
   const leakyGold = 'SECRET-GOLD'
@@ -141,8 +210,11 @@ async function main(): Promise<void> {
   if (runtime.openSandboxRun.toString().includes('beforeStart')) {
     // The default shot path supports benchmark-owned box setup/extract without real sandbox infra.
     const order: string[] = []
+    let createdOptions: unknown
+    let controlCredential: string | undefined
     const fakeClient = {
-      async create() {
+      async create(options: unknown) {
+        createdOptions = options
         return {
           id: 'box-default-shot',
           async exec(command: string, options?: { sessionId?: string }) {
@@ -151,7 +223,9 @@ async function main(): Promise<void> {
           },
           async *streamPrompt(_prompt: string, options?: { sessionId?: string }) {
             order.push(`stream:session=${options?.sessionId ? 'yes' : 'no'}`)
+            yield { type: 'llm_call', data: { tokensIn: 23, tokensOut: 7, costUsd: 0.04 } }
             yield { type: 'result', data: { finalText: 'fallback text' } }
+            yield { type: 'done', data: { outcome: { type: 'completed' } } }
           },
           async delete() {
             order.push('delete')
@@ -175,11 +249,19 @@ async function main(): Promise<void> {
       benchmarks: ['boxy'],
       cells: [{ label: 'default-shot', model: 'm', backend: 'sandbox' }],
       routerBaseUrl: 'x',
-      routerKey: 'x',
+      routerKey: 'sandbox-control-token',
+      modelApiKey: 'model-grant-token',
       resolveAdapter: () => boxAdapter,
-      resolveClient: () => fakeClient as never,
+      resolveClient: (options) => {
+        controlCredential = options.routerKey
+        return fakeClient as never
+      },
     })
     assert.equal(boxy.rows[0]!.resolveRate, 1, 'boxExtract artifact is judged instead of fallback text')
+    assert.equal(boxy.perTask[0]?.artifact, 'PATCH')
+    assert.deepEqual(boxy.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 })
+    assert.equal(controlCredential, 'sandbox-control-token', 'inference grant never authorizes sandbox control')
+    assert.equal((createdOptions as { backend: { model: { apiKey: string } } }).backend.model.apiKey, 'model-grant-token')
     assert.deepEqual(
       order.slice(0, 3),
       ['exec:setup-repo:streams=0:session=yes', 'stream:session=yes', 'exec:extract-patch:streams=1:session=yes'],