@tangle-network/agent-bench 0.9.4 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.9.4",
+  "version": "0.10.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.176.0 <0.177.0",
-    "@tangle-network/agent-interface": "^2.5.0",
-    "@tangle-network/agent-knowledge": "^15.0.0",
+    "@tangle-network/agent-eval": ">=0.178.0 <0.179.0",
+    "@tangle-network/agent-interface": "^2.6.0",
+    "@tangle-network/agent-knowledge": "^15.0.1",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.202.0"
+    "@tangle-network/agent-runtime": "^0.203.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/run-package-tests.mjs

@@ -1,5 +1,6 @@
 import { execFile } from 'node:child_process'
-import { access, readdir, readFile } from 'node:fs/promises'
+import { access, readdir, readFile, realpath } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
@@ -64,6 +65,15 @@ export async function run(command, args, env = process.env) {
   }
 }
 
+export async function runPythonTests(python, env = process.env) {
+  // Fixture roots must be physical paths; the production boundary rejects symlinked ancestors.
+  const physicalTemp = await realpath(env.TMPDIR || tmpdir())
+  await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'], {
+    ...env,
+    TMPDIR: physicalTemp,
+  })
+}
+
 async function main() {
   const python = path.join(benchDir, '.venv', 'bin', 'python')
   try {
@@ -101,7 +111,7 @@ async function main() {
     await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
   }
 
-  await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
+  await runPythonTests(python)
 
   console.log(
     `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,

package/scripts/run-package-tests.test.mjs

@@ -1,6 +1,9 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { packageNodeTestArgs, resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs'
+import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { packageNodeTestArgs, resolvePackageTestTimeoutMs, run, runPythonTests } from './run-package-tests.mjs'
 
 test('package test concurrency reaches Node without changing selected files', () => {
   const files = ['src/first.test.mts', 'src/second.test.ts']
@@ -45,3 +48,25 @@ test('package test timeout reaches the child process', async () => {
   )
   assert.ok(Date.now() - startedAt < 2_000)
 })
+
+test('Python fixtures use physical temporary paths without relaxing symlink guards', async () => {
+  const root = await mkdtemp(path.join(await realpath(tmpdir()), 'bench-python-temp-'))
+  try {
+    const target = path.join(root, 'physical')
+    const linked = path.join(root, 'linked')
+    const recorded = path.join(root, 'recorded.json')
+    const launcher = path.join(root, 'python-fixture')
+    await mkdir(target)
+    await symlink(target, linked, 'dir')
+    await writeFile(launcher, `#!${process.execPath}
+const fs = require('node:fs'); fs.writeFileSync(process.env.RECORDED, JSON.stringify({ temporary: process.env.TMPDIR, args: process.argv.slice(2) }))
+`, { mode: 0o755 })
+    await runPythonTests(launcher, { ...process.env, TMPDIR: linked, RECORDED: recorded })
+    assert.deepEqual(JSON.parse(await readFile(recorded, 'utf8')), {
+      temporary: target,
+      args: ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'],
+    })
+  } finally {
+    await rm(root, { recursive: true, force: true })
+  }
+})

package/src/run-benchmarks.test.mts

@@ -77,6 +77,22 @@ async function main(): Promise<void> {
   assert.equal(judgeFailure.perTask[0]?.ok, false)
   assert.equal(judgeFailure.perTask[0]?.artifact, 'PATCH', 'a judge outage retains the completed agent artifact')
   assert.equal(judgeFailure.perTask[0]?.usage?.costUsd, 0.04, 'a judge outage retains already incurred usage')
+  assert.equal(judgeFailure.perTask[0]?.measurement, 'unavailable')
+  assert.deepEqual(judgeFailure.perTask[0]?.execution, { phase: 'started', terminalOutcome: 'succeeded' })
+
+  const failedExecution = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'paid-failure', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', n: 1, resolveAdapter: resolveStub,
+    runShot: async () => ({
+      artifact: '', ok: false, artifactAvailable: true,
+      execution: { phase: 'started', terminalOutcome: 'failed' },
+      usage: { input: 23, output: 7, costUsd: 0.04 },
+    }),
+  })
+  assert.equal(failedExecution.perTask[0]?.measurement, 'available')
+  assert.equal(failedExecution.perTask[0]?.resolved, false)
+  assert.equal(failedExecution.rows[0]?.errored, 0, 'a measured paid failure stays in the comparison')
+  assert.equal(failedExecution.perTask[0]?.usage?.input, 23)
 
   const controller = new AbortController()
   let started = 0
@@ -93,6 +109,7 @@ async function main(): Promise<void> {
   })
   assert.equal(started, 1, 'cancellation prevents every queued model call')
   assert.equal(cancelled.perTask.length, 4, 'cancelled work remains visible')
+  assert.equal(cancelled.perTask.filter((row) => row.execution?.phase === 'not-started').length, 3)
   assert.equal(cancelled.perTask[0]?.usage?.costUsd, 0.01)
 
   const row = (b: string, c: string) => report.rows.find((r) => r.benchmark === b && r.cell === c)!
@@ -159,6 +176,20 @@ async function main(): Promise<void> {
     input: 11, output: 3, costUsd: 0.02, tokensKnown: false, usdKnown: false,
   }, 'an unreported retry preserves the measured floor without claiming complete accounting')
 
+  const measuredFailedRetry = await runBenchmarks({
+    benchmarks: ['alpha'], cells: [{ label: 'failed-retries', model: 'm' }],
+    routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: resolveStub, n: 1, loopAttempts: 2,
+    runShot: async ({ attempt }) => ({
+      artifact: attempt === 1 ? '' : 'WRONG', ok: false, artifactAvailable: attempt === 2,
+      execution: { phase: 'started', terminalOutcome: 'failed' },
+      usage: { input: 11, output: 3, costUsd: 0.02 },
+    }),
+  })
+  assert.equal(measuredFailedRetry.perTask[0]?.artifact, 'WRONG')
+  assert.equal(measuredFailedRetry.perTask[0]?.measurement, 'available')
+  assert.equal(measuredFailedRetry.rows[0]?.errored, 0, 'a measured failed retry stays in the comparison')
+  assert.deepEqual(measuredFailedRetry.perTask[0]?.usage, { input: 22, output: 6, costUsd: 0.04 })
+
   let validAttempts = 0
   const validRetry = await runBenchmarks({
     benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }],
@@ -212,6 +243,10 @@ async function main(): Promise<void> {
     const order: string[] = []
     let createdOptions: unknown
     let controlCredential: string | undefined
+    let completed = true
+    let text = 'fallback text'
+    let streamThrows = false
+    let setupFails = false
     const fakeClient = {
       async create(options: unknown) {
         createdOptions = options
@@ -219,13 +254,14 @@ async function main(): Promise<void> {
           id: 'box-default-shot',
           async exec(command: string, options?: { sessionId?: string }) {
             order.push(`exec:${command}:streams=${order.filter((x) => x.startsWith('stream:')).length}:session=${options?.sessionId ? 'yes' : 'no'}`)
-            return { exitCode: 0, stdout: command === 'extract-patch' ? 'PATCH' : '', stderr: '' }
+            return { exitCode: setupFails && command === 'setup-repo' ? 1 : 0, stdout: command === 'extract-patch' ? 'PATCH' : '', stderr: '' }
           },
           async *streamPrompt(_prompt: string, options?: { sessionId?: string }) {
             order.push(`stream:session=${options?.sessionId ? 'yes' : 'no'}`)
             yield { type: 'llm_call', data: { tokensIn: 23, tokensOut: 7, costUsd: 0.04 } }
-            yield { type: 'result', data: { finalText: 'fallback text' } }
-            yield { type: 'done', data: { outcome: { type: 'completed' } } }
+            if (streamThrows) throw new Error('stream disconnected')
+            yield { type: 'result', data: { finalText: text, success: completed, status: completed ? 'success' : 'failed' } }
+            yield { type: 'done', data: { outcome: { type: completed ? 'completed' : 'failed' } } }
           },
           async delete() {
             order.push('delete')
@@ -260,6 +296,8 @@ async function main(): Promise<void> {
     assert.equal(boxy.rows[0]!.resolveRate, 1, 'boxExtract artifact is judged instead of fallback text')
     assert.equal(boxy.perTask[0]?.artifact, 'PATCH')
     assert.deepEqual(boxy.perTask[0]?.usage, { input: 23, output: 7, costUsd: 0.04 })
+    assert.deepEqual(boxy.perTask[0]?.execution, { phase: 'started', terminalOutcome: 'succeeded' })
+    assert.equal(boxy.perTask[0]?.measurement, 'available')
     assert.equal(controlCredential, 'sandbox-control-token', 'inference grant never authorizes sandbox control')
     assert.equal((createdOptions as { backend: { model: { apiKey: string } } }).backend.model.apiKey, 'model-grant-token')
     assert.deepEqual(
@@ -267,6 +305,49 @@ async function main(): Promise<void> {
       ['exec:setup-repo:streams=0:session=yes', 'stream:session=yes', 'exec:extract-patch:streams=1:session=yes'],
       'setup runs before the prompt stream, extract runs after the prompt stream, both in the same session',
     )
+    for (const failure of ['stream', 'parser'] as const) {
+      streamThrows = failure === 'stream'
+      const failed = await runBenchmarks({
+        benchmarks: ['boxy'], cells: [{ label: failure, model: 'm', backend: 'sandbox' }],
+        routerBaseUrl: 'x', routerKey: 'x',
+        resolveAdapter: () => ({ ...boxAdapter, output: { parse: () => { throw new Error('parser failed') } } }),
+        resolveClient: () => fakeClient as never,
+      })
+      assert.equal(failed.perTask[0]?.measurement, 'unavailable')
+      assert.equal(failed.perTask[0]?.execution?.phase, 'started')
+      assert.deepEqual(failed.perTask[0]?.usage, {
+        input: 23, output: 7, costUsd: 0.04, tokensKnown: false, usdKnown: false,
+      }, `${failure} failure retains paid receipts without claiming complete accounting`)
+      assert.equal(failed.perTask[0]?.events?.length, failure === 'stream' ? 1 : 3, 'observed events are retained once')
+      assert.match(failed.perTask[0]?.detail ?? '', failure === 'stream' ? /stream disconnected/ : /parser failed/)
+    }
+    streamThrows = false
+    setupFails = true
+    const setupFailure = await runBenchmarks({
+      benchmarks: ['boxy'], cells: [{ label: 'setup-failure', model: 'm', backend: 'sandbox' }],
+      routerBaseUrl: 'x', routerKey: 'x', resolveAdapter: () => boxAdapter,
+      resolveClient: () => fakeClient as never,
+    })
+    assert.deepEqual(setupFailure.perTask[0]?.execution, { phase: 'unknown', terminalOutcome: 'unknown' })
+    assert.equal(setupFailure.perTask[0]?.measurement, 'unavailable')
+    assert.equal(setupFailure.perTask[0]?.usage?.tokensKnown, false, 'no observed receipt does not prove zero usage')
+    assert.equal(setupFailure.perTask[0]?.usage?.usdKnown, false)
+    assert.equal(setupFailure.perTask[0]?.events?.length, 0)
+    setupFails = false
+    for (const success of [false, true]) {
+      completed = success
+      text = ''
+      const empty = await runBenchmarks({
+        benchmarks: ['alpha'], cells: [{ label: 'empty', model: 'm', backend: 'sandbox' }],
+        routerBaseUrl: 'x', routerKey: 'x', n: 1, resolveAdapter: resolveStub,
+        resolveClient: () => fakeClient as never,
+      })
+      assert.equal(empty.perTask[0]?.ok, false)
+      assert.equal(empty.perTask[0]?.measurement, 'available', 'captured empty output is a measurable failure')
+      assert.equal(empty.perTask[0]?.execution?.terminalOutcome, success ? 'succeeded' : 'failed')
+      assert.equal(empty.rows[0]?.errored, 0)
+      assert.equal(empty.perTask[0]?.usage?.input, 23)
+    }
   }
 
   // An unavailable benchmark (preflight throws) is skipped, not fatal; the sweep still runs the rest.

package/src/run-benchmarks.ts

@@ -29,6 +29,7 @@
  */
 
 import { mkdirSync, writeFileSync } from 'node:fs'
+import type { RunTerminalOutcome } from '@tangle-network/agent-eval'
 import type {
   AgentProfile,
   AgentRunSpec,
@@ -68,6 +69,13 @@ export interface BenchShotResult {
   /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
   readonly usage?: ReturnType<typeof sumSandboxUsage>
   readonly events?: readonly SandboxEvent[]
+  /** Observed dispatch and terminal state, independent of artifact quality. */
+  readonly execution?: {
+    readonly phase: 'not-started' | 'started' | 'unknown'
+    readonly terminalOutcome: RunTerminalOutcome
+  }
+  /** Whether the artifact was captured without a read or extraction failure. */
+  readonly artifactAvailable?: boolean
 }
 
 /** Runs one (adapter, task, cell) shot. Defaults to `openSandboxRun`. */
@@ -137,9 +145,11 @@ export interface BenchCellTaskResult {
   readonly rep: number
   readonly resolved: boolean
   readonly score: number
-  /** false = the shot threw or produced no artifact (infra/empty), excluded from the resolve
-   *  denominator so a harness outage can't masquerade as a 0% capability result. */
+  /** Whether execution completed successfully and produced a readable, nonempty artifact. */
   readonly ok: boolean
+  readonly execution?: BenchShotResult['execution']
+  /** Available failed attempts remain in comparisons; unavailable measurement is reported separately. */
+  readonly measurement?: 'available' | 'unavailable'
   readonly detail?: string
   readonly wallMs: number
   /** Exact bytes given to the benchmark judge, retained even when judging fails. */
@@ -237,11 +247,13 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   }
   const controller = new AbortController()
   const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : undefined
+  const observedEvents: SandboxEvent[] = []
   const runOptions: OpenSandboxRunOptions = {
     agentRun,
     signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
     runId: `bench:${adapter.name}:${task.id}:${uniq}`,
     scenarioId: task.id,
+    onSandboxEvent: (event) => { observedEvents.push(event) },
   }
   const boxSetup = adapter.boxSetup
   if (boxSetup) {
@@ -262,8 +274,15 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   let result: BenchShotResult = { artifact: '', ok: false }
   try {
     run = await openSandboxRun(client, runOptions, deliverable)
+    result = { ...result, execution: { phase: 'unknown', terminalOutcome: 'unknown' } }
     const turn = await run.start(prompt ?? task.prompt)
-    result = { artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events }
+    result = {
+      artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events,
+      execution: {
+        phase: 'started',
+        terminalOutcome: turn.outcome.success ? 'succeeded' : turn.outcome.status === 'failed' ? 'failed' : 'incomplete',
+      },
+    }
     // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK.
     let artifact = (turn.out ?? '').trim()
     let boxExtractError: string | undefined
@@ -336,16 +355,26 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     result = {
       artifact,
       ok: turn.outcome.success && artifact.length > 0 && turn.readError === undefined && boxExtractError === undefined,
+      execution: result.execution,
+      artifactAvailable: turn.readError === undefined && boxExtractError === undefined,
       usage: result.usage,
       events: turn.events,
       ...(detail ? { detail } : {}),
     }
   } catch (err) {
+    const events = err instanceof SandboxRunAbortError ? err.events : observedEvents
     result = {
       ...result,
       ok: false,
+      artifactAvailable: false,
+      execution: {
+        phase: events.length > 0 ? 'started' : 'unknown',
+        terminalOutcome: result.execution?.terminalOutcome ?? 'unknown',
+      },
       detail: err instanceof Error ? err.message : String(err),
-      ...(err instanceof SandboxRunAbortError ? { usage: sumSandboxUsage(err.events), events: err.events } : {}),
+      // A thrown capture cannot establish that every paid receipt arrived.
+      usage: { ...sumSandboxUsage(events), tokensKnown: false, usdKnown: false },
+      events,
     }
   } finally {
     if (timer) clearTimeout(timer)
@@ -451,6 +480,8 @@ async function loopedShot(
     return {
       artifact: completed.at(-1)?.artifact ?? '',
       ok: false,
+      execution: pendingShot ? { phase: 'unknown', terminalOutcome: 'unknown' } : completed.at(-1)?.execution,
+      artifactAvailable: false,
       usage: combinedUsage(pendingShot ? [...completed, { artifact: '', ok: false }] : completed),
       events: completed.flatMap((shot) => shot.events ?? []),
       detail: err instanceof Error ? err.message : String(err),
@@ -458,8 +489,10 @@ async function loopedShot(
   }
 
   const best = result.rounds.reduce((winner, candidate) => {
-    if (shots.get(candidate.round)?.ok !== true) return winner
-    if (shots.get(winner.round)?.ok !== true) return candidate
+    const candidateShot = shots.get(candidate.round)
+    const winnerShot = shots.get(winner.round)
+    const rank = (shot: BenchShotResult | undefined) => shot?.ok ? 2 : shot?.artifactAvailable ? 1 : 0
+    if (rank(candidateShot) !== rank(winnerShot)) return rank(candidateShot) > rank(winnerShot) ? candidate : winner
     const a = scores.get(winner.round)
     const b = scores.get(candidate.round)
     if (!a) return candidate
@@ -472,6 +505,8 @@ async function loopedShot(
   return {
     artifact: best.artifact,
     ok: shots.get(best.round)?.ok === true && best.artifact.trim().length > 0,
+    execution: shots.get(best.round)?.execution,
+    artifactAvailable: shots.get(best.round)?.artifactAvailable,
     usage: combinedUsage([...shots.values()]),
     events: [...shots.values()].flatMap((shot) => shot.events ?? []),
     detail: JSON.stringify({
@@ -588,6 +623,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
     const startedAt = Date.now()
     let result: BenchCellTaskResult
     let out: BenchShotResult | undefined
+    let invoked = false
     try {
       opts.signal?.throwIfAborted()
       const shotInput = {
@@ -604,6 +640,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ...(opts.signal ? { signal: opts.signal } : {}),
         ...(opts.resolveClient ? { resolveClient: opts.resolveClient } : {}),
       }
+      invoked = true
       out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
       const score: BenchScore = await job.adapter.judge(job.task, out.artifact)
       result = {
@@ -614,6 +651,8 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         resolved: out.ok && score.resolved,
         score: out.ok ? score.score : 0,
         ok: out.ok,
+        execution: out.execution ?? { phase: out.ok ? 'started' : 'unknown', terminalOutcome: out.ok ? 'succeeded' : 'unknown' },
+        measurement: (out.artifactAvailable ?? out.ok) ? 'available' : 'unavailable',
         ...(out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {}),
         wallMs: Date.now() - startedAt,
         artifact: out.artifact,
@@ -621,8 +660,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ...(out.events === undefined ? {} : { events: out.events }),
       }
     } catch (err) {
-      // A thrown shot/judge is infra error for THIS cell-task: ok=false excludes it from the
-      // resolve denominator (never a silent 0% that hides a harness outage).
+      // Missing results do not prove that dispatch or paid inference never occurred.
       result = {
         benchmark: job.benchmark,
         cell: job.cell.label,
@@ -631,6 +669,11 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         resolved: false,
         score: 0,
         ok: false,
+        execution: out?.execution ?? {
+          phase: !invoked ? 'not-started' : out?.ok ? 'started' : 'unknown',
+          terminalOutcome: out?.ok ? 'succeeded' : 'unknown',
+        },
+        measurement: 'unavailable',
         detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
         wallMs: Date.now() - startedAt,
         ...(out === undefined ? {} : { artifact: out.artifact }),
@@ -660,7 +703,7 @@ function aggregate(perTask: readonly BenchCellTaskResult[]): BenchLeaderboardRow
     const key = `${r.benchmark}\u0000${r.cell}`
     const e = byKey.get(key) ?? { benchmark: r.benchmark, cell: r.cell, n: 0, resolved: 0, errored: 0, scoreSum: 0 }
     e.n += 1
-    if (!r.ok) e.errored += 1
+    if ((r.measurement ?? (r.ok ? 'available' : 'unavailable')) === 'unavailable') e.errored += 1
     else {
       if (r.resolved) e.resolved += 1
       e.scoreSum += r.score