@tangle-network/agent-bench 0.9.4 → 0.11.0

package/src/run-benchmarks.ts

@@ -29,11 +29,14 @@
  */
 
 import { mkdirSync, writeFileSync } from 'node:fs'
+import type { RunTerminalOutcome } from '@tangle-network/agent-eval'
 import type {
   AgentProfile,
   AgentRunSpec,
   Deliverable,
   OpenSandboxRunOptions,
+  SandboxRun,
+  TurnResult,
 } from '@tangle-network/agent-runtime/kernel'
 import { openSandboxRun, SandboxRunAbortError, sumSandboxUsage } from '@tangle-network/agent-runtime/kernel'
 import type { SandboxEvent } from '@tangle-network/sandbox'
@@ -60,6 +63,34 @@ export interface BenchCell {
   readonly profile?: AgentProfile
 }
 
+/** Caller-owned work inside a managed benchmark shot. Final grading stays outside this callback. */
+export interface BenchExecutionContext {
+  readonly prompt: string
+  readonly profile: AgentProfile
+  readonly benchmark: string
+  readonly taskId: string
+  readonly attempt: number
+  readonly signal: AbortSignal
+  /** Runtime owns session continuity. Bench owns capture, extraction, and close. */
+  readonly run: Pick<SandboxRun<string>, 'start' | 'resume' | 'box' | 'sessionId'>
+}
+
+export type BenchExecution = (context: BenchExecutionContext) => Promise<void>
+
+/** One submitted sandbox prompt, including partial evidence when its invocation throws. */
+export interface BenchPromptResult {
+  readonly attempt: number
+  readonly index: number
+  readonly method: 'start' | 'resume'
+  readonly prompt: string
+  readonly sessionId?: string
+  readonly outcome?: TurnResult<string>['outcome']
+  readonly events: readonly SandboxEvent[]
+  readonly usage: ReturnType<typeof sumSandboxUsage>
+  readonly readError?: string
+  readonly error?: string
+}
+
 /** A worker's artifact and observed execution evidence, before external grading. */
 export interface BenchShotResult {
   readonly artifact: string
@@ -68,6 +99,14 @@ export interface BenchShotResult {
   /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
   readonly usage?: ReturnType<typeof sumSandboxUsage>
   readonly events?: readonly SandboxEvent[]
+  readonly prompts?: readonly BenchPromptResult[]
+  /** Observed dispatch and terminal state, independent of artifact quality. */
+  readonly execution?: {
+    readonly phase: 'not-started' | 'started' | 'unknown'
+    readonly terminalOutcome: RunTerminalOutcome
+  }
+  /** Whether the artifact was captured without a read or extraction failure. */
+  readonly artifactAvailable?: boolean
 }
 
 /** Runs one (adapter, task, cell) shot. Defaults to `openSandboxRun`. */
@@ -79,6 +118,8 @@ export type BenchShot = (input: {
   readonly prompt?: string
   /** 1-based attempt index for looped runs. */
   readonly attempt?: number
+  /** Custom shots own whether they consume this managed execution callback. */
+  readonly execute?: BenchExecution
   readonly routerBaseUrl: string
   readonly routerKey: string
   /** Optional inference credential for the box; routerKey continues to authorize sandbox control. */
@@ -123,6 +164,8 @@ export interface RunBenchmarksOptions {
   /** Self-verify each benchmark's judge against its gold artifact on the first task before spending
    *  model tokens; a benchmark whose judge rejects its own gold is recorded unavailable. Default true. */
   readonly verifyJudge?: boolean
+  /** Caller policy inside each managed shot, including every refine-loop attempt. */
+  readonly execute?: BenchExecution
   /** Test seam: a deterministic shot runner. Defaults to the `openSandboxRun` leaf. */
   readonly runShot?: BenchShot
   /** Test seam: resolve a benchmark key to an adapter. Defaults to the registry `resolveAdapter`. */
@@ -137,9 +180,11 @@ export interface BenchCellTaskResult {
   readonly rep: number
   readonly resolved: boolean
   readonly score: number
-  /** false = the shot threw or produced no artifact (infra/empty), excluded from the resolve
-   *  denominator so a harness outage can't masquerade as a 0% capability result. */
+  /** Whether execution completed successfully and produced a readable, nonempty artifact. */
   readonly ok: boolean
+  readonly execution?: BenchShotResult['execution']
+  /** Available failed attempts remain in comparisons; unavailable measurement is reported separately. */
+  readonly measurement?: 'available' | 'unavailable'
   readonly detail?: string
   readonly wallMs: number
   /** Exact bytes given to the benchmark judge, retained even when judging fails. */
@@ -147,6 +192,7 @@ export interface BenchCellTaskResult {
   readonly usage?: ReturnType<typeof sumSandboxUsage>
   /** Worker events only; benchmark grading remains outside this trace. */
   readonly events?: readonly SandboxEvent[]
+  readonly prompts?: readonly BenchPromptResult[]
 }
 
 export interface BenchLeaderboardRow {
@@ -194,7 +240,7 @@ function finalText(events: readonly SandboxEvent[]): string {
 
 /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable
  *  extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */
-const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
+const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, attempt = 1, execute, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
   signal?.throwIfAborted()
   const client = (resolveClient ?? resolveBenchClient)({
     backend: cell.backend ?? 'router',
@@ -237,11 +283,13 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   }
   const controller = new AbortController()
   const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : undefined
+  const observedEvents: SandboxEvent[] = []
   const runOptions: OpenSandboxRunOptions = {
     agentRun,
     signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
     runId: `bench:${adapter.name}:${task.id}:${uniq}`,
     scenarioId: task.id,
+    onSandboxEvent: (event) => { observedEvents.push(event) },
   }
   const boxSetup = adapter.boxSetup
   if (boxSetup) {
@@ -260,10 +308,96 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   }
   let run: Awaited<ReturnType<typeof openSandboxRun<string>>> | undefined
   let result: BenchShotResult = { artifact: '', ok: false }
+  const prompts: BenchPromptResult[] = []
+  const execution: {
+    lastTurn?: TurnResult<string>
+    active?: Promise<TurnResult<string>>
+    accepting: boolean
+    coordinationError?: Error
+  } = { accepting: true }
   try {
     run = await openSandboxRun(client, runOptions, deliverable)
-    const turn = await run.start(prompt ?? task.prompt)
-    result = { artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events }
+    result = { ...result, execution: { phase: 'unknown', terminalOutcome: 'unknown' } }
+    const managedRun = run
+    const invoke = (method: 'start' | 'resume', input: string): Promise<TurnResult<string>> => {
+      if (!execution.accepting || execution.active) {
+        execution.coordinationError = new Error(!execution.accepting
+          ? 'benchmark execution callback has settled'
+          : 'benchmark execution requires sequential start/resume calls')
+        throw execution.coordinationError
+      }
+      runOptions.signal.throwIfAborted()
+      const index = prompts.length
+      const eventStart = observedEvents.length
+      execution.lastTurn = undefined
+      const capture = async (): Promise<TurnResult<string>> => {
+        let turn: TurnResult<string> | undefined
+        let failure: unknown
+        try {
+          turn = await managedRun[method](input)
+          return turn
+        } catch (error) {
+          failure = error
+          throw error
+        } finally {
+          // The observer captures generic stream/parser failures as well as aborts.
+          const events = observedEvents.slice(eventStart)
+          if (turn) execution.lastTurn = { ...turn, events, outcome: { ...turn.outcome } }
+          let sessionId: string | undefined
+          try { sessionId = managedRun.sessionId } catch { /* Creation may have failed. */ }
+          const readError = turn?.readError ?? (failure instanceof SandboxRunAbortError ? failure.readError : undefined)
+          prompts.push({
+            attempt, index, method, prompt: input,
+            ...(sessionId === undefined ? {} : { sessionId }),
+            ...(turn === undefined ? {} : { outcome: { ...turn.outcome } }),
+            events,
+            usage: turn === undefined
+              ? { ...sumSandboxUsage(events), tokensKnown: false, usdKnown: false }
+              : sumSandboxUsage(events),
+            ...(readError === undefined ? {} : { readError }),
+            ...(turn !== undefined ? {} : { error: failure instanceof Error ? failure.message : String(failure) }),
+          })
+        }
+      }
+      const pending = capture()
+      execution.active = pending
+      // Observe even a caller's unawaited invocation; cleanup must await its capture.
+      void pending.then(() => { execution.active = undefined }, () => { execution.active = undefined })
+      return pending
+    }
+    const context: BenchExecutionContext = {
+      prompt: prompt ?? task.prompt, profile: structuredClone(profile), benchmark: adapter.name, taskId: task.id, attempt,
+      signal: runOptions.signal,
+      run: {
+        start: (input) => invoke('start', input),
+        resume: (input) => invoke('resume', input),
+        get box() { return managedRun.box },
+        get sessionId() { return managedRun.sessionId },
+      },
+    }
+    try {
+      await executeWithSignal(async () => {
+        if (execute) await execute(context)
+        else await context.run.start(context.prompt)
+      }, runOptions.signal)
+    } catch (error) {
+      if (execution.active) controller.abort()
+      throw error
+    } finally {
+      execution.accepting = false
+      await execution.active?.catch(() => undefined)
+    }
+    if (execution.coordinationError) throw execution.coordinationError
+    runOptions.signal.throwIfAborted()
+    const turn = execution.lastTurn
+    if (!turn) throw new Error(prompts.at(-1)?.error ?? 'benchmark execution returned without a completed prompt')
+    result = {
+      artifact: '', ok: false, usage: promptUsage(prompts), events: observedEvents, prompts,
+      execution: {
+        phase: 'started',
+        terminalOutcome: turn.outcome.success ? 'succeeded' : turn.outcome.status === 'failed' ? 'failed' : 'incomplete',
+      },
+    }
     // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK.
     let artifact = (turn.out ?? '').trim()
     let boxExtractError: string | undefined
@@ -336,16 +470,27 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     result = {
       artifact,
       ok: turn.outcome.success && artifact.length > 0 && turn.readError === undefined && boxExtractError === undefined,
+      execution: result.execution,
+      artifactAvailable: turn.readError === undefined && boxExtractError === undefined,
       usage: result.usage,
-      events: turn.events,
+      events: observedEvents,
+      prompts,
       ...(detail ? { detail } : {}),
     }
   } catch (err) {
+    const events = observedEvents
     result = {
       ...result,
       ok: false,
+      artifactAvailable: false,
+      execution: {
+        phase: events.length > 0 ? 'started' : 'unknown',
+        terminalOutcome: result.execution?.terminalOutcome ?? 'unknown',
+      },
       detail: err instanceof Error ? err.message : String(err),
-      ...(err instanceof SandboxRunAbortError ? { usage: sumSandboxUsage(err.events), events: err.events } : {}),
+      usage: promptUsage(prompts),
+      events,
+      prompts,
     }
   } finally {
     if (timer) clearTimeout(timer)
@@ -359,6 +504,25 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   return result
 }
 
+/** Stop waiting on policy work when cancelled; the caller must also cancel its external effects. */
+async function executeWithSignal(execute: () => Promise<void>, signal: AbortSignal): Promise<void> {
+  signal.throwIfAborted()
+  let onAbort: (() => void) | undefined
+  const aborted = new Promise<never>((_resolve, reject) => {
+    onAbort = () => reject(signal.reason ?? new Error('aborted'))
+    signal.addEventListener('abort', onAbort, { once: true })
+  })
+  try {
+    await Promise.race([Promise.resolve().then(execute), aborted])
+  } finally {
+    if (onAbort) signal.removeEventListener('abort', onAbort)
+  }
+}
+
+function promptUsage(prompts: readonly BenchPromptResult[]): ReturnType<typeof sumSandboxUsage> {
+  return combinedUsage(prompts.map((prompt) => ({ artifact: '', ok: false, usage: prompt.usage })))
+}
+
 function parseMaybeJson(value: string): unknown {
   try {
     return JSON.parse(value) as unknown
@@ -451,15 +615,20 @@ async function loopedShot(
     return {
       artifact: completed.at(-1)?.artifact ?? '',
       ok: false,
+      execution: pendingShot ? { phase: 'unknown', terminalOutcome: 'unknown' } : completed.at(-1)?.execution,
+      artifactAvailable: false,
       usage: combinedUsage(pendingShot ? [...completed, { artifact: '', ok: false }] : completed),
       events: completed.flatMap((shot) => shot.events ?? []),
+      prompts: completed.flatMap((shot) => shot.prompts ?? []),
       detail: err instanceof Error ? err.message : String(err),
     }
   }
 
   const best = result.rounds.reduce((winner, candidate) => {
-    if (shots.get(candidate.round)?.ok !== true) return winner
-    if (shots.get(winner.round)?.ok !== true) return candidate
+    const candidateShot = shots.get(candidate.round)
+    const winnerShot = shots.get(winner.round)
+    const rank = (shot: BenchShotResult | undefined) => shot?.ok ? 2 : shot?.artifactAvailable ? 1 : 0
+    if (rank(candidateShot) !== rank(winnerShot)) return rank(candidateShot) > rank(winnerShot) ? candidate : winner
     const a = scores.get(winner.round)
     const b = scores.get(candidate.round)
     if (!a) return candidate
@@ -472,8 +641,11 @@ async function loopedShot(
   return {
     artifact: best.artifact,
     ok: shots.get(best.round)?.ok === true && best.artifact.trim().length > 0,
+    execution: shots.get(best.round)?.execution,
+    artifactAvailable: shots.get(best.round)?.artifactAvailable,
     usage: combinedUsage([...shots.values()]),
     events: [...shots.values()].flatMap((shot) => shot.events ?? []),
+    prompts: [...shots.values()].flatMap((shot) => shot.prompts ?? []),
     detail: JSON.stringify({
       mode: 'refine-loop',
       attempts: result.rounds.length,
@@ -588,6 +760,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
     const startedAt = Date.now()
     let result: BenchCellTaskResult
     let out: BenchShotResult | undefined
+    let invoked = false
     try {
       opts.signal?.throwIfAborted()
       const shotInput = {
@@ -603,7 +776,9 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}),
         ...(opts.signal ? { signal: opts.signal } : {}),
         ...(opts.resolveClient ? { resolveClient: opts.resolveClient } : {}),
+        ...(opts.execute ? { execute: opts.execute } : {}),
       }
+      invoked = true
       out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
       const score: BenchScore = await job.adapter.judge(job.task, out.artifact)
       result = {
@@ -614,15 +789,17 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         resolved: out.ok && score.resolved,
         score: out.ok ? score.score : 0,
         ok: out.ok,
+        execution: out.execution ?? { phase: out.ok ? 'started' : 'unknown', terminalOutcome: out.ok ? 'succeeded' : 'unknown' },
+        measurement: (out.artifactAvailable ?? out.ok) ? 'available' : 'unavailable',
         ...(out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {}),
         wallMs: Date.now() - startedAt,
         artifact: out.artifact,
         ...(out.usage === undefined ? {} : { usage: out.usage }),
         ...(out.events === undefined ? {} : { events: out.events }),
+        ...(out.prompts === undefined ? {} : { prompts: out.prompts }),
       }
     } catch (err) {
-      // A thrown shot/judge is infra error for THIS cell-task: ok=false excludes it from the
-      // resolve denominator (never a silent 0% that hides a harness outage).
+      // Missing results do not prove that dispatch or paid inference never occurred.
       result = {
         benchmark: job.benchmark,
         cell: job.cell.label,
@@ -631,11 +808,17 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         resolved: false,
         score: 0,
         ok: false,
+        execution: out?.execution ?? {
+          phase: !invoked ? 'not-started' : out?.ok ? 'started' : 'unknown',
+          terminalOutcome: out?.ok ? 'succeeded' : 'unknown',
+        },
+        measurement: 'unavailable',
         detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
         wallMs: Date.now() - startedAt,
         ...(out === undefined ? {} : { artifact: out.artifact }),
         ...(out?.usage === undefined ? {} : { usage: out.usage }),
         ...(out?.events === undefined ? {} : { events: out.events }),
+        ...(out?.prompts === undefined ? {} : { prompts: out.prompts }),
       }
     }
     void index
@@ -660,7 +843,7 @@ function aggregate(perTask: readonly BenchCellTaskResult[]): BenchLeaderboardRow
     const key = `${r.benchmark}\u0000${r.cell}`
     const e = byKey.get(key) ?? { benchmark: r.benchmark, cell: r.cell, n: 0, resolved: 0, errored: 0, scoreSum: 0 }
     e.n += 1
-    if (!r.ok) e.errored += 1
+    if ((r.measurement ?? (r.ok ? 'available' : 'unavailable')) === 'unavailable') e.errored += 1
     else {
       if (r.resolved) e.resolved += 1
       e.scoreSum += r.score