@tangle-network/agent-bench 0.8.19 → 0.8.22

package/dist/benchmarks/tau-bench-shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"tau-bench-shared.js","names":[],"sources":["../../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA2CA,MAAa,mBAA0C,EACrD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,2CAA2C,CAC/D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;CACvF,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,GAAG,UAAU,YAAY,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;CAC5D,OAAO,QAAQ,IAAI,OAAO;AAC5B;AAEA,SAAS,YAAY,QAAgC;CACnD,OAAO,QAAQ,IAAI,OAAO,cAAc,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;CACjF,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,cAAc,IAAI;EAClB,aAAa,IAAI;EACjB,oBAAoB,IAAI;CAC1B;CACA,OAAO;EACL,IAAI,IAAI;EACR;EACA,QAAQ;GACN,OAAO;GACP,iCAAiC,IAAI,OAAO;GAC5C;GACA;GACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;GAC3G;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;CAC7D,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAC/D,MAAM,IAAI,MAAM,GAAG,UAAU,QAAQ,KAAK,GAAG,kDAAkD;CAEjG,OAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;CAC3D,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAClG,OAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;CAC3F,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;CAClE,QAAQ,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,YAAY,eAAe,KAAK,OAAO,kBAAkB;CACjG,OAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;CAmB9G,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;GAAQ;EAAC;EAlB7B,YAAY,MAkB4B;EAAG,KAAK,SAAS;CAAE,CAAC;CAC3E,OAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;CAoB1H,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAM,KAAK;EAAQ;CAAY,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,sBAAsB,QAA0C;CAC9E,MAAM,eAAe,QAAQ,IAAI,OAAO,iBAAiB;CAEzD,OAAO;EACL,MAAM,OAAO;EACb,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,qBAAqB,OAAO,aAAa;GAE5E,MAAM,WAAW,GAAG,IAAI,wBAAwB,gBAAgB,OAAO,IAAI;GAC3E,MAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;EACnD;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,QAAQ,IAAI;GAClD,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,gCAAgC,OAAO,KAAK,OAAO;GAC9F,OAAO,kBAAkB,QAAQ,KAAK,IAAI;EAC5C;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,wBAAwB,OAAO,KAAK,sBAAsB;GACrG,MAAM,OAAO,SAAS,MAAM,OAAO,IAAI;GACvC,MAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;GAC5C,MAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;GAC7E,MAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;GACpE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,QAAQ,KAAK;KAAQ,OAAO,OAAO;IAAM,CAAC;GAC1F;EACF;CACF;AACF"}
+{"version":3,"file":"tau-bench-shared.js","names":[],"sources":["../../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n  /** Git commit of the upstream checkout the task was loaded from. Official loads always carry it; fixtures never do. */\n  upstreamCommit?: string\n  /** Installed `tau2` distribution version in the interpreter that loaded the task. */\n  upstreamVersion?: string\n}\n\n/** The exact upstream identity a tau task or score is bound to. */\nexport interface TauUpstreamPin {\n  upstreamCommit: string\n  upstreamVersion: string\n}\n\n/**\n * Resolve the upstream checkout's git identity. Throws when the directory is\n * not a git checkout or holds uncommitted changes, because a commit over dirty\n * state does not identify the code that produced the tasks or the score.\n */\nasync function resolveUpstreamCommit(root: string, benchName: string): Promise<string> {\n  let head: string\n  try {\n    const { stdout } = await execFileAsync('git', ['-C', root, 'rev-parse', 'HEAD'])\n    head = stdout.trim()\n  } catch (err) {\n    throw new Error(\n      `${benchName}: cannot resolve the upstream commit of ${root} (${err instanceof Error ? err.message : err}). ` +\n        'The bench dir must be a git checkout of https://github.com/sierra-research/tau2-bench.',\n    )\n  }\n  const { stdout: status } = await execFileAsync('git', ['-C', root, 'status', '--porcelain'])\n  if (status.trim().length > 0) {\n    const dirty = status.trim().split('\\n').slice(0, 5).join(', ')\n    throw new Error(\n      `${benchName}: upstream checkout ${root} has uncommitted changes (${dirty}). ` +\n        'Commit or stash them so the pinned commit identifies the benchmark code.',\n    )\n  }\n  return head\n}\n\n/**\n * Refuse to score a task against a checkout that moved after the task was\n * loaded: the reward recomputation would run under a different upstream\n * identity than the one stamped on the task.\n */\nexport function assertTauUpstreamPinUnchanged(\n  benchName: string,\n  taskId: string,\n  recorded: Readonly<{ upstreamCommit?: string; upstreamVersion?: string }>,\n  live: TauUpstreamPin,\n): void {\n  if (recorded.upstreamCommit !== undefined && recorded.upstreamCommit !== live.upstreamCommit) {\n    throw new Error(\n      `${benchName} task ${taskId} was loaded from upstream commit ${recorded.upstreamCommit} ` +\n        `but the checkout is now at ${live.upstreamCommit}; reload tasks before judging.`,\n    )\n  }\n  if (recorded.upstreamVersion !== undefined && recorded.upstreamVersion !== live.upstreamVersion) {\n    throw new Error(\n      `${benchName} task ${taskId} was loaded with tau2 ${recorded.upstreamVersion} ` +\n        `but the interpreter now holds tau2 ${live.upstreamVersion}; reload tasks before judging.`,\n    )\n  }\n}\n\nconst upstreamVersionSnippet = `\nimport sys\nsys.dont_write_bytecode = True\nimport importlib.metadata\ntry:\n    upstream_version = importlib.metadata.version(\"tau2\")\nexcept importlib.metadata.PackageNotFoundError:\n    raise SystemExit(\n        \"installed tau2 distribution not found in this interpreter; \"\n        \"run 'uv sync' inside the tau2-bench checkout and set \"\n        \"AGENT_BENCH_PYTHON to <checkout>/.venv/bin/python3\"\n    )\n`\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string, pin?: TauUpstreamPin): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n    ...(pin ?? {}),\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string, pin?: TauUpstreamPin): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split, pin))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\n${upstreamVersionSnippet}\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps({\"upstreamVersion\": upstream_version, \"rows\": rows}))\n`\n  const upstreamCommit = await resolveUpstreamCommit(root, config.name)\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  const report = JSON.parse(stdout) as { upstreamVersion?: unknown; rows?: unknown }\n  if (typeof report.upstreamVersion !== 'string' || report.upstreamVersion.length === 0 || !Array.isArray(report.rows)) {\n    throw new Error(`${config.name}: upstream task loader returned no pinned version/rows`)\n  }\n  const pin: TauUpstreamPin = { upstreamCommit, upstreamVersion: report.upstreamVersion }\n  return selectRows(report.rows as TauRow[], opts, config, opts.split, pin)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\n${upstreamVersionSnippet}\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores, \"upstreamVersion\": upstream_version}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const upstreamCommit = await resolveUpstreamCommit(dir, config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const upstreamVersion = report.upstreamVersion\n      if (typeof upstreamVersion !== 'string' || upstreamVersion.length === 0) {\n        throw new Error(`${config.name}: reward recomputation returned no pinned tau2 version`)\n      }\n      const live: TauUpstreamPin = { upstreamCommit, upstreamVersion }\n      assertTauUpstreamPinUnchanged(config.name, meta.taskId, meta, live)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          domain: meta.domain,\n          count: report.count,\n          upstreamCommit,\n          upstreamVersion,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,gBAAgB,UAAU,QAAQ;;;;;;AA6CxC,eAAe,sBAAsB,MAAc,WAAoC;CACrF,IAAI;CACJ,IAAI;EACF,MAAM,EAAE,WAAW,MAAM,cAAc,OAAO;GAAC;GAAM;GAAM;GAAa;EAAM,CAAC;EAC/E,OAAO,OAAO,KAAK;CACrB,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,GAAG,UAAU,0CAA0C,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0FAE3G;CACF;CACA,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,OAAO;EAAC;EAAM;EAAM;EAAU;CAAa,CAAC;CAC3F,IAAI,OAAO,KAAK,CAAC,CAAC,SAAS,GAAG;EAC5B,MAAM,QAAQ,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI;EAC7D,MAAM,IAAI,MACR,GAAG,UAAU,sBAAsB,KAAK,4BAA4B,MAAM,4EAE5E;CACF;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,8BACd,WACA,QACA,UACA,MACM;CACN,IAAI,SAAS,mBAAmB,KAAA,KAAa,SAAS,mBAAmB,KAAK,gBAC5E,MAAM,IAAI,MACR,GAAG,UAAU,QAAQ,OAAO,mCAAmC,SAAS,eAAe,8BACvD,KAAK,eAAe,+BACtD;CAEF,IAAI,SAAS,oBAAoB,KAAA,KAAa,SAAS,oBAAoB,KAAK,iBAC9E,MAAM,IAAI,MACR,GAAG,UAAU,QAAQ,OAAO,wBAAwB,SAAS,gBAAgB,sCACrC,KAAK,gBAAgB,+BAC/D;AAEJ;AAEA,MAAM,yBAAyB;;;;;;;;;;;;;AAc/B,MAAa,mBAA0C,EACrD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,2CAA2C,CAC/D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;CACvF,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,GAAG,UAAU,YAAY,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;CAC5D,OAAO,QAAQ,IAAI,OAAO;AAC5B;AAEA,SAAS,YAAY,QAAgC;CACnD,OAAO,QAAQ,IAAI,OAAO,cAAc,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAAgB,KAAiC;CACvG,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,cAAc,IAAI;EAClB,aAAa,IAAI;EACjB,oBAAoB,IAAI;EACxB,GAAI,OAAO,CAAC;CACd;CACA,OAAO;EACL,IAAI,IAAI;EACR;EACA,QAAQ;GACN,OAAO;GACP,iCAAiC,IAAI,OAAO;GAC5C;GACA;GACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;GAC3G;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;CAC7D,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAC/D,MAAM,IAAI,MAAM,GAAG,UAAU,QAAQ,KAAK,GAAG,kDAAkD;CAEjG,OAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAAgB,KAAmC;CAChI,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,QAAQ,OAAO,GAAG,CAAC;CAChE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAClG,OAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;CAC3F,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;CAClE,QAAQ,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,YAAY,eAAe,KAAK,OAAO,kBAAkB;CACjG,OAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;CAC9G,MAAM,SAAS,YAAY,MAAM;CACjC,MAAM,SAAS;;;;;;;EAOf,uBAAuB;;;;;;;;;;;CAWvB,MAAM,iBAAiB,MAAM,sBAAsB,MAAM,OAAO,IAAI;CACpE,MAAM,SAAS,MAAM,cAAc,QAAQ;EAAC;EAAM;EAAQ,KAAK,SAAS;CAAE,CAAC;CAC3E,MAAM,SAAS,KAAK,MAAM,MAAM;CAChC,IAAI,OAAO,OAAO,oBAAoB,YAAY,OAAO,gBAAgB,WAAW,KAAK,CAAC,MAAM,QAAQ,OAAO,IAAI,GACjH,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,uDAAuD;CAExF,MAAM,MAAsB;EAAE;EAAgB,iBAAiB,OAAO;CAAgB;CACtF,OAAO,WAAW,OAAO,MAAkB,MAAM,QAAQ,KAAK,OAAO,GAAG;AAC1E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;CAqB1H,MAAM,SAAS,MAAM,cAAc;;;;;;;EAbnC,uBAAuB;;;;;;;;;;;;GAaoB;EAAC;EAAM,KAAK;EAAQ;CAAY,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,sBAAsB,QAA0C;CAC9E,MAAM,eAAe,QAAQ,IAAI,OAAO,iBAAiB;CAEzD,OAAO;EACL,MAAM,OAAO;EACb,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,qBAAqB,OAAO,aAAa;GAE5E,MAAM,WAAW,GAAG,IAAI,wBAAwB,gBAAgB,OAAO,IAAI;GAC3E,MAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;EACnD;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,QAAQ,IAAI;GAClD,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,gCAAgC,OAAO,KAAK,OAAO;GAC9F,OAAO,kBAAkB,QAAQ,KAAK,IAAI;EAC5C;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,wBAAwB,OAAO,KAAK,sBAAsB;GACrG,MAAM,OAAO,SAAS,MAAM,OAAO,IAAI;GACvC,MAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;GAC5C,MAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;GAC7E,MAAM,iBAAiB,MAAM,sBAAsB,KAAK,OAAO,IAAI;GACnE,MAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;GACpE,MAAM,kBAAkB,OAAO;GAC/B,IAAI,OAAO,oBAAoB,YAAY,gBAAgB,WAAW,GACpE,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,uDAAuD;GAExF,MAAM,OAAuB;IAAE;IAAgB;GAAgB;GAC/D,8BAA8B,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI;GAClE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,QAAQ,KAAK;KACb,OAAO,OAAO;KACd;KACA;IACF,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/types.d.ts

@@ -1,4 +1,4 @@
-import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
+import { AgentTurnUsage, OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/types.d.ts
 interface BenchTask {
   /** Stable benchmark instance id. */
@@ -41,6 +41,11 @@ interface BenchScore {
   detail?: string;
   /** Present only when the caller explicitly requested durable judge evidence. */
   judgeArtifacts?: JudgeArtifactReceipt;
+  /** The judge model turn's exact Runtime usage record, present only when the
+   *  adapter's judge is itself a model call. `usdKnown: false` and
+   *  `tokensKnown: false` survive verbatim — an unknown judge cost stays
+   *  unknown instead of reading as zero. Deterministic judges never set it. */
+  judgeUsage?: AgentTurnUsage;
 }
 interface LoadOptions {
   limit?: number;

package/dist/router-turn-uTYO6KQ1.js.map

@@ -1 +1 @@
-{"version":3,"file":"router-turn-uTYO6KQ1.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type AgentTurnInput,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turnInput: AgentTurnInput =\n    typeof input === 'string'\n      ? { prompt: input }\n      : {\n          providerOptions: {\n            messages: input.messages.map((message) => ({ ...message })),\n          },\n        }\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      turnInput,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA4GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,UAAU,eAAe;EAC7B,SAAS;EACT,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;CAChD,CAAC;CACD,MAAM,YACJ,OAAO,UAAU,WACb,EAAE,QAAQ,MAAM,IAChB,EACE,iBAAiB,EACf,UAAU,MAAM,SAAS,KAAK,aAAa,EAAE,GAAG,QAAQ,EAAE,EAC5D,EACF;CACN,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY;EAAS,SAAS,OAAO;CAAQ,GACrD,WACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}
+{"version":3,"file":"router-turn-uTYO6KQ1.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type AgentTurnInput,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      // A bench cap bounds the VISIBLE answer, which the Router lowers as `max_tokens`.\n      ...(settings.maxTokens !== undefined\n        ? { maxVisibleOutputTokens: settings.maxTokens }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turnInput: AgentTurnInput =\n    typeof input === 'string'\n      ? { prompt: input }\n      : {\n          providerOptions: {\n            messages: input.messages.map((message) => ({ ...message })),\n          },\n        }\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      turnInput,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA+GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,UAAU,eAAe;EAC7B,SAAS;EACT,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;CAChD,CAAC;CACD,MAAM,YACJ,OAAO,UAAU,WACb,EAAE,QAAQ,MAAM,IAChB,EACE,iBAAiB,EACf,UAAU,MAAM,SAAS,KAAK,aAAa,EAAE,GAAG,QAAQ,EAAE,EAC5D,EACF;CACN,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY;EAAS,SAAS,OAAO;CAAQ,GACrD,WACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.19",
+  "version": "0.8.22",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.149.0 <0.150.0",
-    "@tangle-network/agent-interface": "^1.3.0",
-    "@tangle-network/agent-knowledge": "^8.0.10",
+    "@tangle-network/agent-eval": ">=0.163.2 <0.164.0",
+    "@tangle-network/agent-interface": "^1.4.0",
+    "@tangle-network/agent-knowledge": "^10.7.0",
     "@tangle-network/sandbox": ">=0.31.0 <0.32.0",
-    "@tangle-network/agent-runtime": "^0.143.0"
+    "@tangle-network/agent-runtime": "^0.156.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/benchmarks/dabstep-official.test.mts

@@ -0,0 +1,117 @@
+/**
+ * Official DABStep path tests against a real Python interpreter. The staged
+ * checkout carries the exact official layout (splits/, files/, grade.py) while
+ * the released dataset.csv lives apart and is reached through the
+ * DABSTEP_DATASET_CSV override. grade.py is a deterministic stub here; the
+ * judge bridge, CSV parsing, split filtering, and override plumbing are the
+ * production paths. AGENT_BENCH_PYTHON is fixed before the adapter modules
+ * load because the harness resolves its interpreter at import time.
+ */
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import test from 'node:test'
+
+const python = execFileSync('python3', ['-c', 'import sys; print(sys.executable)'], { encoding: 'utf8' }).trim()
+process.env.AGENT_BENCH_PYTHON = python
+
+const { createDabstepAdapter } = await import('./dabstep')
+
+const datasetCsv = [
+  'task_id,question,guidelines,all_golds_by_task',
+  `7,What is the calibration answer?,Answer with the exact integer.,"[{'kind': 'number', 'value': 42.0}]"`,
+  `8,Which scheme?,Answer with the scheme name.,"[{'kind': 'scheme', 'value': 'nexpay'}]"`,
+  '',
+].join('\n')
+
+const gradeStub = [
+  'def grade(pred, golds):',
+  '    text = str(pred).strip()',
+  '    for gold in golds:',
+  '        value = gold["value"]',
+  '        if text == str(value):',
+  '            return True',
+  '        try:',
+  '            if float(text) == float(value):',
+  '                return True',
+  '        except (TypeError, ValueError):',
+  '            pass',
+  '    return False',
+  '',
+].join('\n')
+
+async function makeCheckout(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'dabstep-official-'))
+  await mkdir(join(dir, 'splits'), { recursive: true })
+  await mkdir(join(dir, 'files'), { recursive: true })
+  await writeFile(join(dir, 'splits', 'easy.txt'), '7\n8\n')
+  await writeFile(join(dir, 'files', 'payments-readme.md'), 'synthetic payment files')
+  await writeFile(join(dir, 'grade.py'), gradeStub)
+  return dir
+}
+
+async function withEnv<T>(patch: Record<string, string | undefined>, fn: () => Promise<T>): Promise<T> {
+  const old: Record<string, string | undefined> = {}
+  for (const [key, value] of Object.entries(patch)) {
+    old[key] = process.env[key]
+    if (value === undefined) delete process.env[key]
+    else process.env[key] = value
+  }
+  try {
+    return await fn()
+  } finally {
+    for (const [key, value] of Object.entries(old)) {
+      if (value === undefined) delete process.env[key]
+      else process.env[key] = value
+    }
+  }
+}
+
+test('DABSTEP_DATASET_CSV lets the checkout and the released rows live apart, end to end through grade.py', async () => {
+  const dir = await makeCheckout()
+  const dataDir = await mkdtemp(join(tmpdir(), 'dabstep-rows-'))
+  const dataset = join(dataDir, 'released-dataset.csv')
+  await writeFile(dataset, datasetCsv)
+  try {
+    await withEnv(
+      { DABSTEP_FIXTURES: undefined, DABSTEP_DIR: dir, DABSTEP_DATASET_CSV: dataset },
+      async () => {
+        const adapter = createDabstepAdapter()
+        await adapter.preflight()
+        const tasks = await adapter.loadTasks({ split: 'easy' })
+        assert.deepEqual(
+          tasks.map((task) => task.id),
+          ['7', '8'],
+        )
+        const meta = tasks[0].metadata as Record<string, unknown>
+        assert.equal(meta.resourceRoot, join(dir, 'files'))
+        assert.equal(await adapter.goldArtifact(tasks[0]), '42')
+        const pass = await adapter.judge(tasks[0], '42')
+        assert.equal(pass.resolved, true)
+        assert.equal(pass.score, 1)
+        const fail = await adapter.judge(tasks[0], 'dog')
+        assert.equal(fail.resolved, false)
+        assert.equal(fail.score, 0)
+      },
+    )
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+    await rm(dataDir, { recursive: true, force: true })
+  }
+})
+
+test('a relative DABSTEP_DATASET_CSV is refused before any load', async () => {
+  const dir = await makeCheckout()
+  try {
+    await withEnv(
+      { DABSTEP_FIXTURES: undefined, DABSTEP_DIR: dir, DABSTEP_DATASET_CSV: 'relative/dataset.csv' },
+      async () => {
+        await assert.rejects(() => createDabstepAdapter().preflight(), /DABSTEP_DATASET_CSV must be an absolute path/)
+      },
+    )
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+})

package/src/benchmarks/dabstep.ts

@@ -3,13 +3,17 @@
  * payment files. Worker artifact = final answer text. Judge = the official
  * DABStep `grade.py` normalization/matching function. No LLM judge.
  *
- * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
- * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
- * exposes `metadata.resourceRoot` so runners can mount the benchmark files into
- * AgentProfile.resources.files; it does not paste the dataset into prompt text.
+ * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout
+ * (https://github.com/EnvCommons/DABStep) for `splits/*.txt`, `files/*`, and
+ * `grade.py`, plus the released `dataset.csv` task rows — beside the checkout
+ * or anywhere via `DABSTEP_DATASET_CSV`. The checkout does not ship
+ * `dataset.csv`; see the preflight error text for where the rows come from.
+ * The adapter exposes `metadata.resourceRoot` so runners can mount the
+ * benchmark files into AgentProfile.resources.files; it does not paste the
+ * dataset into prompt text.
  */
 
-import { join } from 'node:path'
+import { isAbsolute, join } from 'node:path'
 import { access, readFile, stat } from 'node:fs/promises'
 import type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'
 import { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'
@@ -35,6 +39,31 @@ const dabstepDir = (): string | undefined => process.env.DABSTEP_DIR
 const gradeFile = (dir: string): string => join(dir, 'grade.py')
 const resourceRoot = (dir: string): string => join(dir, 'files')
 
+/**
+ * Where the released task rows come from. The EnvCommons/DABStep git checkout
+ * carries grade.py, splits/, and files/ but not dataset.csv: the row file
+ * (task_id,question,guidelines,all_golds_by_task) ships with the OpenReward
+ * DABStep environment, mounted at /orwd_data/dataset.csv on that platform.
+ * The public https://huggingface.co/datasets/adyen/DABstep release holds the
+ * same task rows without golds (data/tasks/all.jsonl; answers withheld for the
+ * leaderboard) and a 10-task dev split with reference answers
+ * (data/tasks/dev.jsonl).
+ */
+const datasetAcquisitionHint =
+  'The DABStep checkout ships grade.py, splits/, and files/ but not dataset.csv. ' +
+  'dataset.csv (task_id,question,guidelines,all_golds_by_task) is distributed with the OpenReward DABStep environment ' +
+  '(mounted at /orwd_data/dataset.csv on that platform); the public https://huggingface.co/datasets/adyen/DABstep release ' +
+  'carries the task rows without golds (data/tasks/all.jsonl) and a 10-task dev split with reference answers (data/tasks/dev.jsonl). ' +
+  'Place dataset.csv beside the checkout, or point DABSTEP_DATASET_CSV at it.'
+
+/** The released row file: `<dir>/dataset.csv`, or the `DABSTEP_DATASET_CSV` override so checkout and export can live apart. */
+function datasetFile(dir: string): string {
+  const override = process.env.DABSTEP_DATASET_CSV
+  if (override === undefined) return join(dir, 'dataset.csv')
+  if (!isAbsolute(override)) throw new Error('DABSTEP_DATASET_CSV must be an absolute path')
+  return override
+}
+
 async function assertFile(path: string, label: string): Promise<void> {
   try {
     await access(path)
@@ -44,7 +73,14 @@ async function assertFile(path: string, label: string): Promise<void> {
 }
 
 async function assertOfficialFiles(dir: string, split: string): Promise<void> {
-  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')
+  const dataset = datasetFile(dir)
+  try {
+    await access(dataset)
+  } catch (err) {
+    throw new Error(
+      `DABStep: missing released dataset.csv at ${dataset} (${err instanceof Error ? err.message : err}). ${datasetAcquisitionHint}`,
+    )
+  }
   await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)
   await assertFile(gradeFile(dir), 'official grade.py')
   const files = resourceRoot(dir)
@@ -124,10 +160,10 @@ root = Path(sys.argv[1])
 split = sys.argv[2]
 limit = None if sys.argv[3] == "" else int(sys.argv[3])
 ids = set(json.loads(sys.argv[4]))
-dataset = root / "dataset.csv"
+dataset = Path(sys.argv[5])
 split_file = root / "splits" / f"{split}.txt"
 if not dataset.exists():
-    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}")
+    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}; the checkout does not ship it — see the adapter preflight error for acquisition")
 if not split_file.exists():
     raise SystemExit(f"missing official DABStep split file at {split_file}")
 split_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}
@@ -150,7 +186,13 @@ if not out:
     raise SystemExit(f"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}")
 print(json.dumps(out))
 `
-  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])
+  const stdout = await runVenvPython(script, [
+    dir,
+    split,
+    opts.limit === undefined ? '' : String(opts.limit),
+    JSON.stringify(opts.ids ?? []),
+    datasetFile(dir),
+  ])
   return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)
 }
 
@@ -166,7 +208,8 @@ export function createDabstepAdapter(): BenchmarkAdapter {
       const dir = dabstepDir()
       if (!dir) {
         throw new Error(
-          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',
+          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep and set DABSTEP_DIR=/path/to/DABStep. ' +
+            datasetAcquisitionHint,
         )
       }
       await assertOfficialFiles(dir, DEFAULT_SPLIT)

package/src/benchmarks/external-adapters.test.mts

@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { createAgentBenchAdapter } from './agentbench'
 import { createBfclAdapter } from './bfcl'
-import { createFinResearchBenchAdapter } from './finresearchbench'
+import { createFinResearchBenchAdapter, scoreOfficialJudgeTurn } from './finresearchbench'
 import { createTau2BenchAdapter } from './tau2-bench'
 import { createTau3BankingAdapter } from './tau3-banking'
 import { createToolLlmAdapter } from './toollm'
@@ -148,3 +148,56 @@ test('FinResearchBench fixture mode is explicit and live mode requires exported
     await assert.rejects(() => createFinResearchBenchAdapter().preflight(), /FINRESEARCHBENCH_DATA_FILE is required/)
   })
 })
+
+const officialFinResearchTask = {
+  id: 'row-1',
+  prompt: 'unused',
+  split: 'macro',
+  metadata: {
+    id: 'row-1',
+    category: 'macro',
+    question: 'Assess margin trajectory.',
+    referenceAnswer: 'Margin expansion',
+    referenceReport: 'Margin expansion',
+    logicTree: null,
+    rubric: null,
+    scoring: 'official-logic-tree-judge',
+  },
+}
+
+test('FinResearchBench official judge score carries the judge turn usage verbatim', () => {
+  const usage = {
+    input: 1200,
+    output: 44,
+    costUsd: 0.0123,
+    model: 'deepseek-v4-flash',
+  }
+  const score = scoreOfficialJudgeTurn(
+    officialFinResearchTask,
+    { finalText: '```json\n{"score": 8}\n```', usage },
+    'deepseek-v4-flash',
+  )
+  assert.equal(score.score, 0.8)
+  assert.equal(score.resolved, true)
+  assert.deepEqual(score.judgeUsage, usage)
+  const detail = JSON.parse(score.detail ?? '{}') as Record<string, unknown>
+  assert.equal(detail.judgeModel, 'deepseek-v4-flash')
+  assert.equal(detail.scoring, 'official-logic-tree-judge')
+})
+
+test('FinResearchBench keeps unknown judge cost unknown instead of reporting zero spend', () => {
+  const usage = {
+    input: 0,
+    output: 0,
+    tokensKnown: false as const,
+    usdKnown: false as const,
+    estimatedCostUsd: 0.002,
+  }
+  const score = scoreOfficialJudgeTurn(officialFinResearchTask, { finalText: '{"score": 3}', usage }, 'deepseek-v4-flash')
+  assert.equal(score.score, 0.3)
+  assert.equal(score.resolved, false)
+  assert.equal(score.judgeUsage?.usdKnown, false)
+  assert.equal(score.judgeUsage?.tokensKnown, false)
+  assert.equal(score.judgeUsage?.costUsd, undefined)
+  assert.equal(score.judgeUsage?.estimatedCostUsd, 0.002)
+})

package/src/benchmarks/finresearchbench.ts

@@ -9,6 +9,7 @@
 
 import { readFile, stat } from 'node:fs/promises'
 import { join } from 'node:path'
+import type { AgentTurnUsage } from '@tangle-network/agent-runtime/kernel'
 import { runBenchRouterTurn } from '../router-turn'
 import { benchRoot } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
@@ -178,7 +179,30 @@ function parseJudgeScore(content: string): { score: number; raw: unknown } {
   throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)
 }
 
-async function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {
+/**
+ * Map one drained judge turn to a BenchScore. The judge turn's exact Runtime
+ * usage record lands on `judgeUsage` verbatim, so an unproven token count or
+ * dollar amount stays flagged (`tokensKnown`/`usdKnown` false) instead of
+ * reading as a known zero cost.
+ */
+export function scoreOfficialJudgeTurn(
+  task: BenchTask,
+  turn: Readonly<{ finalText?: string; usage: AgentTurnUsage }>,
+  requestedJudgeModel: string,
+): BenchScore {
+  const meta = readMeta(task)
+  const content = turn.finalText
+  if (!content) throw new Error('FinResearchBench judge returned no message content')
+  const { score, raw } = parseJudgeScore(content)
+  return {
+    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
+    score,
+    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: requestedJudgeModel, raw }),
+    judgeUsage: turn.usage,
+  }
+}
+
+async function runOfficialJudge(task: BenchTask, meta: FinResearchMeta, response: string): Promise<BenchScore> {
   if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)
   const router = routerConfig()
   const turn = await runBenchRouterTurn(
@@ -198,14 +222,7 @@ async function runOfficialJudge(meta: FinResearchMeta, response: string): Promis
     },
     fillTemplate(meta, response),
   )
-  const content = turn.finalText
-  if (!content) throw new Error('FinResearchBench judge returned no message content')
-  const { score, raw } = parseJudgeScore(content)
-  return {
-    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
-    score,
-    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),
-  }
+  return scoreOfficialJudgeTurn(task, turn, router.model)
 }
 
 function normalizeText(value: string): string {
@@ -267,7 +284,7 @@ export function createFinResearchBenchAdapter(): BenchmarkAdapter {
           detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),
         }
       }
-      return runOfficialJudge(meta, artifact)
+      return runOfficialJudge(task, meta, artifact)
     },
   }
 }