@tangle-network/agent-bench 0.8.18 → 0.8.22

package/src/benchmarks/tau-bench-shared.ts

@@ -7,12 +7,16 @@
  * choose env names, default domain, and fixture file.
  */
 
+import { execFile } from 'node:child_process'
 import { readFile, stat } from 'node:fs/promises'
 import { resolve } from 'node:path'
+import { promisify } from 'node:util'
 import type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'
 import { runVenvPython } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
+const execFileAsync = promisify(execFile)
+
 export interface TauBenchConfig {
   name: string
   fixturePath: string
@@ -39,8 +43,84 @@ interface TauMeta {
   userScenario?: unknown
   description?: unknown
   evaluationCriteria?: unknown
+  /** Git commit of the upstream checkout the task was loaded from. Official loads always carry it; fixtures never do. */
+  upstreamCommit?: string
+  /** Installed `tau2` distribution version in the interpreter that loaded the task. */
+  upstreamVersion?: string
+}
+
+/** The exact upstream identity a tau task or score is bound to. */
+export interface TauUpstreamPin {
+  upstreamCommit: string
+  upstreamVersion: string
+}
+
+/**
+ * Resolve the upstream checkout's git identity. Throws when the directory is
+ * not a git checkout or holds uncommitted changes, because a commit over dirty
+ * state does not identify the code that produced the tasks or the score.
+ */
+async function resolveUpstreamCommit(root: string, benchName: string): Promise<string> {
+  let head: string
+  try {
+    const { stdout } = await execFileAsync('git', ['-C', root, 'rev-parse', 'HEAD'])
+    head = stdout.trim()
+  } catch (err) {
+    throw new Error(
+      `${benchName}: cannot resolve the upstream commit of ${root} (${err instanceof Error ? err.message : err}). ` +
+        'The bench dir must be a git checkout of https://github.com/sierra-research/tau2-bench.',
+    )
+  }
+  const { stdout: status } = await execFileAsync('git', ['-C', root, 'status', '--porcelain'])
+  if (status.trim().length > 0) {
+    const dirty = status.trim().split('\n').slice(0, 5).join(', ')
+    throw new Error(
+      `${benchName}: upstream checkout ${root} has uncommitted changes (${dirty}). ` +
+        'Commit or stash them so the pinned commit identifies the benchmark code.',
+    )
+  }
+  return head
 }
 
+/**
+ * Refuse to score a task against a checkout that moved after the task was
+ * loaded: the reward recomputation would run under a different upstream
+ * identity than the one stamped on the task.
+ */
+export function assertTauUpstreamPinUnchanged(
+  benchName: string,
+  taskId: string,
+  recorded: Readonly<{ upstreamCommit?: string; upstreamVersion?: string }>,
+  live: TauUpstreamPin,
+): void {
+  if (recorded.upstreamCommit !== undefined && recorded.upstreamCommit !== live.upstreamCommit) {
+    throw new Error(
+      `${benchName} task ${taskId} was loaded from upstream commit ${recorded.upstreamCommit} ` +
+        `but the checkout is now at ${live.upstreamCommit}; reload tasks before judging.`,
+    )
+  }
+  if (recorded.upstreamVersion !== undefined && recorded.upstreamVersion !== live.upstreamVersion) {
+    throw new Error(
+      `${benchName} task ${taskId} was loaded with tau2 ${recorded.upstreamVersion} ` +
+        `but the interpreter now holds tau2 ${live.upstreamVersion}; reload tasks before judging.`,
+    )
+  }
+}
+
+const upstreamVersionSnippet = `
+import sys
+sys.dont_write_bytecode = True
+import importlib.metadata
+try:
+    upstream_version = importlib.metadata.version("tau2")
+except importlib.metadata.PackageNotFoundError:
+    raise SystemExit(
+        "installed tau2 distribution not found in this interpreter; "
+        "run 'uv sync' inside the tau2-bench checkout and set "
+        "AGENT_BENCH_PYTHON to <checkout>/.venv/bin/python3"
+    )
+`
+
 export const tauResultsOutput: OutputAdapter<string> = {
   parse(events) {
     let text = ''
@@ -70,7 +150,7 @@ function benchDomain(config: TauBenchConfig): string {
   return process.env[config.domainEnv] ?? config.defaultDomain
 }
 
-function rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {
+function rowToTask(row: TauRow, config: TauBenchConfig, split?: string, pin?: TauUpstreamPin): BenchTask {
   const meta: TauMeta = {
     taskId: row.id,
     domain: row.domain,
@@ -78,6 +158,7 @@ function rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTa
     userScenario: row.user_scenario,
     description: row.description,
     evaluationCriteria: row.evaluation_criteria,
+    ...(pin ?? {}),
   }
   return {
     id: row.id,
@@ -103,8 +184,8 @@ function readMeta(task: BenchTask, benchName: string): TauMeta {
   return md as unknown as TauMeta
 }
 
-function selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {
-  let tasks = rows.map((row) => rowToTask(row, config, split))
+function selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string, pin?: TauUpstreamPin): BenchTask[] {
+  let tasks = rows.map((row) => rowToTask(row, config, split, pin))
   if (opts.ids) {
     const want = new Set(opts.ids)
     tasks = tasks.filter((task) => want.has(task.id))
@@ -130,6 +211,7 @@ root = Path(sys.argv[1])
 domain = sys.argv[2]
 split = sys.argv[3] or None
 sys.path.insert(0, str(root / "src"))
+${upstreamVersionSnippet}
 from tau2.registry import registry
 loader = registry.get_tasks_loader(domain)
 tasks = loader(split)
@@ -138,10 +220,16 @@ for task in tasks:
     row = task.model_dump(mode="json")
     row["domain"] = domain
     rows.append(row)
-print(json.dumps(rows))
+print(json.dumps({"upstreamVersion": upstream_version, "rows": rows}))
 `
+  const upstreamCommit = await resolveUpstreamCommit(root, config.name)
   const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])
-  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)
+  const report = JSON.parse(stdout) as { upstreamVersion?: unknown; rows?: unknown }
+  if (typeof report.upstreamVersion !== 'string' || report.upstreamVersion.length === 0 || !Array.isArray(report.rows)) {
+    throw new Error(`${config.name}: upstream task loader returned no pinned version/rows`)
+  }
+  const pin: TauUpstreamPin = { upstreamCommit, upstreamVersion: report.upstreamVersion }
+  return selectRows(report.rows as TauRow[], opts, config, opts.split, pin)
 }
 
 async function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {
@@ -152,6 +240,7 @@ root = Path(sys.argv[1])
 task_id = sys.argv[2]
 artifact = Path(sys.argv[3])
 sys.path.insert(0, str(root / "src"))
+${upstreamVersionSnippet}
 from tau2.data_model.simulation import Results
 from tau2.scripts.evaluate_trajectories import compute_simulation_rewards
 results = Results.load(artifact)
@@ -162,7 +251,7 @@ for sim in updated.simulations:
         scores.append(float(sim.reward_info.reward))
 if not scores:
     raise SystemExit(f"no scored simulations for task_id={task_id} in {artifact}")
-print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores}))
+print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores, "upstreamVersion": upstream_version}))
 `
   const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)
   return JSON.parse(stdout.trim().split('\n').at(-1) ?? '{}') as Record<string, unknown>
@@ -202,12 +291,25 @@ export function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter
       const meta = readMeta(task, config.name)
       const artifactPath = resolve(artifact.trim())
       await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)
+      const upstreamCommit = await resolveUpstreamCommit(dir, config.name)
       const report = await scoreOfficialTrajectory(dir, meta, artifactPath)
+      const upstreamVersion = report.upstreamVersion
+      if (typeof upstreamVersion !== 'string' || upstreamVersion.length === 0) {
+        throw new Error(`${config.name}: reward recomputation returned no pinned tau2 version`)
+      }
+      const live: TauUpstreamPin = { upstreamCommit, upstreamVersion }
+      assertTauUpstreamPinUnchanged(config.name, meta.taskId, meta, live)
       const score = typeof report.score === 'number' ? report.score : 0
       return {
         resolved: score === 1,
         score,
-        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),
+        detail: JSON.stringify({
+          taskId: meta.taskId,
+          domain: meta.domain,
+          count: report.count,
+          upstreamCommit,
+          upstreamVersion,
+        }),
       }
     },
   }

package/src/benchmarks/tau-pin.test.mts

@@ -0,0 +1,53 @@
+/**
+ * The two tau upstream-pin refusals that protect a benchmark receipt: a score
+ * must never be attributed to code the checkout no longer holds.
+ */
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import test from 'node:test'
+import { assertTauUpstreamPinUnchanged } from './tau-bench-shared'
+import { createTau3BankingAdapter } from './tau3-banking'
+
+function git(dir: string, ...args: string[]): string {
+  return execFileSync('git', ['-C', dir, ...args], { encoding: 'utf8' }).trim()
+}
+
+test('a dirty upstream checkout is refused before any task load', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'tau-pin-'))
+  const previous = process.env.TAU3_BENCH_DIR
+  const fixtures = process.env.TAU3_FIXTURES
+  try {
+    await writeFile(join(dir, 'placeholder.txt'), 'committed')
+    git(dir, 'init', '--quiet')
+    // Fixture repo: keep machine-wide hooks out of the throwaway commit.
+    git(dir, 'config', 'core.hooksPath', '/dev/null')
+    git(dir, 'add', '-A')
+    git(dir, '-c', 'user.email=bench@test', '-c', 'user.name=bench', 'commit', '--quiet', '-m', 'pin fixture')
+    await writeFile(join(dir, 'placeholder.txt'), 'edited after the commit')
+
+    delete process.env.TAU3_FIXTURES
+    process.env.TAU3_BENCH_DIR = dir
+    await assert.rejects(() => createTau3BankingAdapter().loadTasks({ limit: 1 }), /uncommitted changes/)
+  } finally {
+    if (previous === undefined) delete process.env.TAU3_BENCH_DIR
+    else process.env.TAU3_BENCH_DIR = previous
+    if (fixtures !== undefined) process.env.TAU3_FIXTURES = fixtures
+    await rm(dir, { recursive: true, force: true })
+  }
+})
+
+test('a checkout or interpreter that moved after load is refused at judge time', () => {
+  const live = { upstreamCommit: 'abc', upstreamVersion: '1.0.1' }
+  assertTauUpstreamPinUnchanged('tau3-banking', 'task_001', live, live)
+  assert.throws(
+    () => assertTauUpstreamPinUnchanged('tau3-banking', 'task_001', { upstreamCommit: 'def' }, live),
+    /loaded from upstream commit def/,
+  )
+  assert.throws(
+    () => assertTauUpstreamPinUnchanged('tau3-banking', 'task_001', { upstreamVersion: '0.9.0' }, live),
+    /loaded with tau2 0\.9\.0/,
+  )
+})

package/src/benchmarks/types.ts

@@ -8,7 +8,7 @@
  * from the benchmark's published evaluation harness.
  */
 
-import type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'
+import type { AgentTurnUsage, OutputAdapter } from '@tangle-network/agent-runtime/kernel'
 
 export interface BenchTask {
   /** Stable benchmark instance id. */
@@ -54,6 +54,11 @@ export interface BenchScore {
   detail?: string
   /** Present only when the caller explicitly requested durable judge evidence. */
   judgeArtifacts?: JudgeArtifactReceipt
+  /** The judge model turn's exact Runtime usage record, present only when the
+   *  adapter's judge is itself a model call. `usdKnown: false` and
+   *  `tokensKnown: false` survive verbatim — an unknown judge cost stays
+   *  unknown instead of reading as zero. Deterministic judges never set it. */
+  judgeUsage?: AgentTurnUsage
 }
 
 export interface LoadOptions {

package/src/hev-structural.mts

@@ -358,7 +358,8 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content:
             model: {
               provider: 'tangle-router',
               default: cfg.model,
-              metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens },
+              metadata: { temperature: cfg.temperature },
+              maxVisibleOutputTokens: cfg.maxTokens,
             },
             ...(system ? { prompt: { systemPrompt: system } } : {}),
           },

package/src/mbpp-structural.mts

@@ -339,7 +339,8 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content:
             model: {
               provider: 'tangle-router',
               default: cfg.model,
-              metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens },
+              metadata: { temperature: cfg.temperature },
+              maxVisibleOutputTokens: cfg.maxTokens,
             },
             ...(system ? { prompt: { systemPrompt: system } } : {}),
           },

package/src/official-optimizer-config.mts

@@ -93,10 +93,10 @@ export function officialOptimizerModel(options: {
       provider: options.provider ?? new URL(options.baseUrl).hostname,
       default: options.model,
       ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}),
-      metadata: {
-        maxTokens: options.maxOutputTokensPerRequest,
-        ...(options.temperature !== undefined ? { temperature: options.temperature } : {}),
-      },
+      maxVisibleOutputTokens: options.maxOutputTokensPerRequest,
+      ...(options.temperature !== undefined
+        ? { metadata: { temperature: options.temperature } }
+        : {}),
     },
   }
   const profileDigest = canonicalAgentProfileDigest(profile)

package/src/router-turn.ts

@@ -69,7 +69,6 @@ export function withBenchProfile(
   const metadata = {
     ...(base.model?.metadata ?? {}),
     ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),
-    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),
     ...(settings.retry !== undefined ? { retry: settings.retry } : {}),
     ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),
     ...(settings.seed !== undefined ? { seed: settings.seed } : {}),
@@ -84,6 +83,10 @@ export function withBenchProfile(
       ...(settings.reasoningEffort !== undefined
         ? { reasoningEffort: settings.reasoningEffort }
         : {}),
+      // A bench cap bounds the VISIBLE answer, which the Router lowers as `max_tokens`.
+      ...(settings.maxTokens !== undefined
+        ? { maxVisibleOutputTokens: settings.maxTokens }
+        : {}),
       ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
     },
     ...(settings.systemPrompt !== undefined

package/src/supervisor-arena.mts

@@ -307,7 +307,8 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content:
             model: {
               provider: 'tangle-router',
               default: cfg.model,
-              metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens },
+              metadata: { temperature: cfg.temperature },
+              maxVisibleOutputTokens: cfg.maxTokens,
             },
             ...(system ? { prompt: { systemPrompt: system } } : {}),
           },

package/src/swe-arena/capacity.ts

@@ -141,7 +141,8 @@ export function httpCapacityProbe(spec: HttpProbeSpec): CapacityProbe {
           model: {
             provider: spec.provider,
             default: spec.model,
-            metadata: { temperature: 0, maxTokens: spec.maxTokens ?? 8000 },
+            metadata: { temperature: 0 },
+            maxVisibleOutputTokens: spec.maxTokens ?? 8000,
           },
           prompt: { systemPrompt: 'Reply with the single word OK.' },
         },

package/src/swe-self-improve.mts

@@ -32,7 +32,8 @@ async function main(): Promise<void> {
     model: {
       provider: 'tangle-router',
       default: workerModel,
-      metadata: { maxTokens: 8000, maxTurns: innerTurns },
+      metadata: { maxTurns: innerTurns },
+      maxVisibleOutputTokens: 8000,
     },
   }
   const authorProfile = (model: string, name: string): AgentProfile => ({
@@ -41,7 +42,7 @@ async function main(): Promise<void> {
     model: {
       provider: 'tangle-router',
       default: model,
-      metadata: { maxTokens: 8000 },
+      maxVisibleOutputTokens: 8000,
     },
     prompt: { systemPrompt: strategyAuthorSystemPrompt },
   })

package/src/trata-gepa.mts

@@ -134,7 +134,8 @@ async function chatComplete(
         model: {
           provider: 'tangle-router',
           default: model,
-          metadata: { temperature: 0, maxTokens },
+          metadata: { temperature: 0 },
+          maxVisibleOutputTokens: maxTokens,
         },
         ...(system ? { prompt: { systemPrompt: system } } : {}),
       },

package/src/trata-hedge-solve.mts

@@ -69,7 +69,8 @@ const result = await runBenchRouterTurn(
       model: {
         provider: 'tangle-router',
         default: model,
-        metadata: { temperature, maxTokens },
+        metadata: { temperature },
+        maxVisibleOutputTokens: maxTokens,
       },
       prompt: { systemPrompt: instruction },
     },