@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/tau-bench-shared.js

@@ -1,10 +1,170 @@
-import {
-  createTauBenchAdapter,
-  tauResultsOutput
-} from "../chunk-IZ5M6OAC.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createTauBenchAdapter,
-  tauResultsOutput
-};
+import { runVenvPython } from "./_harness.js";
+import { readFile, stat } from "node:fs/promises";
+import { resolve } from "node:path";
+//#region src/benchmarks/tau-bench-shared.ts
+/**
+* Shared tau-bench adapter spine.
+*
+* tau2 and tau3 live in the same upstream repository/package namespace today:
+* `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation
+* for task loading and reward recomputation so the domain/version adapters only
+* choose env names, default domain, and fixture file.
+*/
+const tauResultsOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:text|path|json)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+async function assertPath(path, label, benchName) {
+	try {
+		await stat(path);
+	} catch (err) {
+		throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+function benchDir(config) {
+	return process.env[config.dirEnv];
+}
+function benchDomain(config) {
+	return process.env[config.domainEnv] ?? config.defaultDomain;
+}
+function rowToTask(row, config, split) {
+	const meta = {
+		taskId: row.id,
+		domain: row.domain,
+		split,
+		userScenario: row.user_scenario,
+		description: row.description,
+		evaluationCriteria: row.evaluation_criteria
+	};
+	return {
+		id: row.id,
+		split,
+		prompt: [
+			config.taskIntro,
+			`Run this task in the official ${row.domain} domain.`,
+			"The benchmark is a simulated multi-turn user/tool conversation.",
+			"",
+			typeof row.user_scenario === "string" ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),
+			"",
+			"Return the path to the official tau results.json or trajectory file containing this task run."
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task, benchName) {
+	const md = task.metadata;
+	if (!md || typeof md.taskId !== "string" || typeof md.domain !== "string") throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts, config, split) {
+	let tasks = rows.map((row) => rowToTask(row, config, split));
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`);
+	return tasks;
+}
+async function loadFixtures(config, opts) {
+	const rows = JSON.parse(await readFile(config.fixturePath, "utf8"));
+	console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`);
+	return selectRows(rows, opts, config, opts.split);
+}
+async function loadOfficialTasks(config, root, opts) {
+	const stdout = await runVenvPython(`
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+domain = sys.argv[2]
+split = sys.argv[3] or None
+sys.path.insert(0, str(root / "src"))
+from tau2.registry import registry
+loader = registry.get_tasks_loader(domain)
+tasks = loader(split)
+rows = []
+for task in tasks:
+    row = task.model_dump(mode="json")
+    row["domain"] = domain
+    rows.append(row)
+print(json.dumps(rows))
+`, [
+		root,
+		benchDomain(config),
+		opts.split ?? ""
+	]);
+	return selectRows(JSON.parse(stdout), opts, config, opts.split);
+}
+async function scoreOfficialTrajectory(root, meta, artifactPath) {
+	const stdout = await runVenvPython(`
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+artifact = Path(sys.argv[3])
+sys.path.insert(0, str(root / "src"))
+from tau2.data_model.simulation import Results
+from tau2.scripts.evaluate_trajectories import compute_simulation_rewards
+results = Results.load(artifact)
+updated = compute_simulation_rewards(results)
+scores = []
+for sim in updated.simulations:
+    if sim.task_id == task_id and sim.reward_info is not None:
+        scores.append(float(sim.reward_info.reward))
+if not scores:
+    raise SystemExit(f"no scored simulations for task_id={task_id} in {artifact}")
+print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores}))
+`, [
+		root,
+		meta.taskId,
+		artifactPath
+	], 0);
+	return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+}
+function createTauBenchAdapter(config) {
+	const fixturesMode = process.env[config.fixturesEnv] === "1";
+	return {
+		name: config.name,
+		output: tauResultsOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = benchDir(config);
+			if (!dir) throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`);
+			await assertPath(`${dir}/src/tau2/registry.py`, "tau registry", config.name);
+			await loadOfficialTasks(config, dir, { limit: 1 });
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(config, opts);
+			const dir = benchDir(config);
+			if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`);
+			return loadOfficialTasks(config, dir, opts);
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			const dir = benchDir(config);
+			if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`);
+			const meta = readMeta(task, config.name);
+			const artifactPath = resolve(artifact.trim());
+			await assertPath(artifactPath, "tau results/trajectory artifact", config.name);
+			const report = await scoreOfficialTrajectory(dir, meta, artifactPath);
+			const score = typeof report.score === "number" ? report.score : 0;
+			return {
+				resolved: score === 1,
+				score,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					domain: meta.domain,
+					count: report.count
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createTauBenchAdapter, tauResultsOutput };
+
 //# sourceMappingURL=tau-bench-shared.js.map

package/dist/benchmarks/tau-bench-shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"tau-bench-shared.js","names":[],"sources":["../../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA2CA,MAAa,mBAA0C,EACrD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,2CAA2C,CAC/D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;CACvF,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,GAAG,UAAU,YAAY,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;CAC5D,OAAO,QAAQ,IAAI,OAAO;AAC5B;AAEA,SAAS,YAAY,QAAgC;CACnD,OAAO,QAAQ,IAAI,OAAO,cAAc,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;CACjF,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,cAAc,IAAI;EAClB,aAAa,IAAI;EACjB,oBAAoB,IAAI;CAC1B;CACA,OAAO;EACL,IAAI,IAAI;EACR;EACA,QAAQ;GACN,OAAO;GACP,iCAAiC,IAAI,OAAO;GAC5C;GACA;GACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;GAC3G;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;CAC7D,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAC/D,MAAM,IAAI,MAAM,GAAG,UAAU,QAAQ,KAAK,GAAG,kDAAkD;CAEjG,OAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;CAC3D,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAClG,OAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;CAC3F,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;CAClE,QAAQ,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,YAAY,eAAe,KAAK,OAAO,kBAAkB;CACjG,OAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;CAmB9G,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;GAAQ;EAAC;EAlB7B,YAAY,MAkB4B;EAAG,KAAK,SAAS;CAAE,CAAC;CAC3E,OAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;CAoB1H,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAM,KAAK;EAAQ;CAAY,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,sBAAsB,QAA0C;CAC9E,MAAM,eAAe,QAAQ,IAAI,OAAO,iBAAiB;CAEzD,OAAO;EACL,MAAM,OAAO;EACb,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,qBAAqB,OAAO,aAAa;GAE5E,MAAM,WAAW,GAAG,IAAI,wBAAwB,gBAAgB,OAAO,IAAI;GAC3E,MAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;EACnD;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,QAAQ,IAAI;GAClD,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,gCAAgC,OAAO,KAAK,OAAO;GAC9F,OAAO,kBAAkB,QAAQ,KAAK,IAAI;EAC5C;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,wBAAwB,OAAO,KAAK,sBAAsB;GACrG,MAAM,OAAO,SAAS,MAAM,OAAO,IAAI;GACvC,MAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;GAC5C,MAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;GAC7E,MAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;GACpE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,QAAQ,KAAK;KAAQ,OAAO,OAAO;IAAM,CAAC;GAC1F;EACF;CACF;AACF"}

package/dist/benchmarks/tau2-bench.d.ts

@@ -1,7 +1,7 @@
-import * as _tangle_network_agent_runtime_loops from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-declare const tau2ResultsOutput: _tangle_network_agent_runtime_loops.OutputAdapter<string>;
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/tau2-bench.d.ts
+declare const tau2ResultsOutput: import("@tangle-network/agent-runtime/loops").OutputAdapter<string>;
 declare function createTau2BenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createTau2BenchAdapter, tau2ResultsOutput };
+//# sourceMappingURL=tau2-bench.d.ts.map

package/dist/benchmarks/tau2-bench.js

@@ -1,11 +1,29 @@
-import {
-  createTau2BenchAdapter,
-  tau2ResultsOutput
-} from "../chunk-RCYQEFNX.js";
-import "../chunk-IZ5M6OAC.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createTau2BenchAdapter,
-  tau2ResultsOutput
-};
+import { benchRoot } from "./_harness.js";
+import { createTauBenchAdapter, tauResultsOutput } from "./tau-bench-shared.js";
+import { join } from "node:path";
+//#region src/benchmarks/tau2-bench.ts
+/**
+* tau2-bench adapter (Sierra tau2/tau-bench successor).
+*
+* Worker artifact = a tau2 `results.json`/trajectory file. Judge = tau2's own
+* reward recomputation over that trajectory. A normal final-answer transcript is
+* not a valid artifact for this benchmark.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "tau2-bench.json");
+const tau2ResultsOutput = tauResultsOutput;
+function createTau2BenchAdapter() {
+	return createTauBenchAdapter({
+		name: "tau2-bench",
+		fixturePath: FIXTURES,
+		fixturesEnv: "TAU2_FIXTURES",
+		dirEnv: "TAU2_BENCH_DIR",
+		domainEnv: "TAU2_DOMAIN",
+		defaultDomain: "retail",
+		taskIntro: "Run this tau2 task in the official tau2 text benchmark.",
+		installHint: "clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench."
+	});
+}
+//#endregion
+export { createTau2BenchAdapter, tau2ResultsOutput };
+
 //# sourceMappingURL=tau2-bench.js.map

package/dist/benchmarks/tau2-bench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"tau2-bench.js","names":[],"sources":["../../src/benchmarks/tau2-bench.ts"],"sourcesContent":["/**\n * tau2-bench adapter (Sierra tau2/tau-bench successor).\n *\n * Worker artifact = a tau2 `results.json`/trajectory file. Judge = tau2's own\n * reward recomputation over that trajectory. A normal final-answer transcript is\n * not a valid artifact for this benchmark.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter, tauResultsOutput } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau2-bench.json')\n\nexport const tau2ResultsOutput = tauResultsOutput\n\nexport function createTau2BenchAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau2-bench',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU2_FIXTURES',\n    dirEnv: 'TAU2_BENCH_DIR',\n    domainEnv: 'TAU2_DOMAIN',\n    defaultDomain: 'retail',\n    taskIntro: 'Run this tau2 task in the official tau2 text benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;;;;AAaA,MAAM,WAAW,KAAK,WAAW,YAAY,iBAAiB;AAE9D,MAAa,oBAAoB;AAEjC,SAAgB,yBAA2C;CACzD,OAAO,sBAAsB;EAC3B,MAAM;EACN,aAAa;EACb,aAAa;EACb,QAAQ;EACR,WAAW;EACX,eAAe;EACf,WAAW;EACX,aACE;CACJ,CAAC;AACH"}

package/dist/benchmarks/tau3-banking.d.ts

@@ -1,15 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * tau3-banking adapter.
- *
- * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with
- * package namespace `tau2` and a new `banking_knowledge` domain. This adapter
- * reuses the shared tau reward recomputation path and only changes the default
- * domain/env names.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/tau3-banking.d.ts
 declare function createTau3BankingAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createTau3BankingAdapter };
+//# sourceMappingURL=tau3-banking.d.ts.map

package/dist/benchmarks/tau3-banking.js

@@ -1,9 +1,29 @@
-import {
-  createTau3BankingAdapter
-} from "../chunk-SFLA7OH3.js";
-import "../chunk-IZ5M6OAC.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createTau3BankingAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { createTauBenchAdapter } from "./tau-bench-shared.js";
+import { join } from "node:path";
+//#region src/benchmarks/tau3-banking.ts
+/**
+* tau3-banking adapter.
+*
+* The current upstream tau3 release is the `sierra-research/tau2-bench` repo with
+* package namespace `tau2` and a new `banking_knowledge` domain. This adapter
+* reuses the shared tau reward recomputation path and only changes the default
+* domain/env names.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "tau3-banking.json");
+function createTau3BankingAdapter() {
+	return createTauBenchAdapter({
+		name: "tau3-banking",
+		fixturePath: FIXTURES,
+		fixturesEnv: "TAU3_FIXTURES",
+		dirEnv: "TAU3_BENCH_DIR",
+		domainEnv: "TAU3_DOMAIN",
+		defaultDomain: "banking_knowledge",
+		taskIntro: "Run this tau3 banking task in the official tau3 knowledge benchmark.",
+		installHint: "clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench."
+	});
+}
+//#endregion
+export { createTau3BankingAdapter };
+
 //# sourceMappingURL=tau3-banking.js.map

package/dist/benchmarks/tau3-banking.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"tau3-banking.js","names":[],"sources":["../../src/benchmarks/tau3-banking.ts"],"sourcesContent":["/**\n * tau3-banking adapter.\n *\n * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with\n * package namespace `tau2` and a new `banking_knowledge` domain. This adapter\n * reuses the shared tau reward recomputation path and only changes the default\n * domain/env names.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau3-banking.json')\n\nexport function createTau3BankingAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau3-banking',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU3_FIXTURES',\n    dirEnv: 'TAU3_BENCH_DIR',\n    domainEnv: 'TAU3_DOMAIN',\n    defaultDomain: 'banking_knowledge',\n    taskIntro: 'Run this tau3 banking task in the official tau3 knowledge benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;;;;;AAcA,MAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,SAAgB,2BAA6C;CAC3D,OAAO,sBAAsB;EAC3B,MAAM;EACN,aAAa;EACb,aAAa;EACb,QAAQ;EACR,WAAW;EACX,eAAe;EACf,WAAW;EACX,aACE;CACJ,CAAC;AACH"}

package/dist/benchmarks/terminal-bench.d.ts

@@ -1,24 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * Terminal-Bench adapter. Each task = a Docker environment + an English
- * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact
- * is the shell script the agent ran to attempt the task; the judge REPLAYS that
- * script in a fresh task container via the Terminal-Bench harness (`tb run` with
- * our ScriptAgent), then the task's own verifier scores the resulting state.
- * Fully deterministic — no LLM judge, no self-authored score.
- *
- * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`
- * installed + a running Docker daemon (per-task images are built on first run).
- * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the
- * Terminal-Bench registry on first run.
- *
- * Process/Docker/report plumbing is shared via ./_harness; this file owns the
- * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay
- * argv, and the results.json shape.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/terminal-bench.d.ts
 declare function createTerminalBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createTerminalBenchAdapter };
+//# sourceMappingURL=terminal-bench.d.ts.map

package/dist/benchmarks/terminal-bench.js

@@ -1,8 +1,141 @@
-import {
-  createTerminalBenchAdapter
-} from "../chunk-XYA4XSNU.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createTerminalBenchAdapter
-};
+import { benchRoot, preflightVenvImports, readJsonReport, runStagedJudge, runVenvPython, safeRunId, stageFile, venvBinAt, venvPythonAt } from "./_harness.js";
+import { join } from "node:path";
+//#region src/benchmarks/terminal-bench.ts
+/**
+* Terminal-Bench adapter. Each task = a Docker environment + an English
+* instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact
+* is the shell script the agent ran to attempt the task; the judge REPLAYS that
+* script in a fresh task container via the Terminal-Bench harness (`tb run` with
+* our ScriptAgent), then the task's own verifier scores the resulting state.
+* Fully deterministic — no LLM judge, no self-authored score.
+*
+* Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`
+* installed + a running Docker daemon (per-task images are built on first run).
+* Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the
+* Terminal-Bench registry on first run.
+*
+* Process/Docker/report plumbing is shared via ./_harness; this file owns the
+* Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay
+* argv, and the results.json shape.
+*/
+const terminalBenchVenvDir = () => process.env.TERMINAL_BENCH_VENV ?? ".venv-terminal-bench";
+const terminalBenchPython = () => venvPythonAt(terminalBenchVenvDir());
+const terminalBenchBin = () => venvBinAt(terminalBenchVenvDir(), "tb");
+const DATASET = "terminal-bench-core";
+const DATASET_VERSION = "0.1.1";
+const DATASET_REF = `${DATASET}==${DATASET_VERSION}`;
+const FIXTURE_IDS = ["hello-world"];
+const SCRIPT_AGENT = "tb_agents.script_agent:ScriptAgent";
+/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on
+*  first run). Reads instruction from each task.yaml and the gold solution. */
+async function loadRows(opts) {
+	const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS);
+	const limit = opts.limit ?? null;
+	const stdout = await runVenvPython(`
+import json, sys
+from pathlib import Path
+from terminal_bench.dataset.dataset import Dataset
+from terminal_bench.handlers.trial_handler import TaskPaths
+
+req_ids = json.loads(sys.argv[1]) if sys.argv[1] else None
+limit = json.loads(sys.argv[2]) if sys.argv[2] else None
+
+ds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)
+
+import yaml
+out = []
+for task_dir in ds:
+    tp = TaskPaths(task_dir)
+    cfg = yaml.safe_load(tp.task_config_path.read_text())
+    try:
+        sol = tp.solution_path.read_text()
+    except FileNotFoundError:
+        sol = None
+    out.append({
+        "id": task_dir.name,
+        "instruction": cfg["instruction"],
+        "task_dir": str(task_dir),
+        "solution": sol if (tp.solution_path.suffix == ".sh" if sol is not None else False) else None,
+    })
+print(json.dumps(out))
+`, [ids ? JSON.stringify(ids) : "", limit !== null ? String(limit) : ""], 0, terminalBenchPython());
+	return JSON.parse(stdout);
+}
+function createTerminalBenchAdapter() {
+	return {
+		name: "terminal-bench",
+		async preflight() {
+			await preflightVenvImports({
+				modules: ["terminal_bench"],
+				requireDocker: true,
+				python: terminalBenchPython(),
+				fix: `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench (an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; override the dir with TERMINAL_BENCH_VENV) ; (2) ensure the Docker daemon is running (the judge builds per-task images on first run). The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`
+			});
+		},
+		async loadTasks(opts = {}) {
+			const rows = await loadRows(opts);
+			if (rows.length === 0) throw new Error(`terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} (dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`);
+			return rows.map((r) => ({
+				id: r.id,
+				split: DATASET_VERSION,
+				prompt: r.instruction,
+				metadata: {
+					dataset: DATASET,
+					datasetVersion: DATASET_VERSION,
+					datasetRef: DATASET_REF,
+					taskDir: r.task_dir,
+					solution: r.solution,
+					instruction: r.instruction
+				}
+			}));
+		},
+		async goldArtifact(task) {
+			const sol = task.metadata?.solution;
+			return typeof sol === "string" ? sol : void 0;
+		},
+		async judge(task, artifact) {
+			const runId = safeRunId("bench", `${task.id}-${Date.now()}`);
+			return runStagedJudge({
+				tmpPrefix: "tbench-",
+				bin: terminalBenchBin(),
+				cwd: () => benchRoot,
+				async stage(dir) {
+					await stageFile(join(dir, "attempt.sh"), artifact);
+				},
+				argv: (dir) => [
+					"run",
+					"-d",
+					DATASET_REF,
+					"-t",
+					task.id,
+					"--agent-import-path",
+					SCRIPT_AGENT,
+					"--agent-kwarg",
+					`script_path=${join(dir, "attempt.sh")}`,
+					"--output-path",
+					join(dir, "runs"),
+					"--run-id",
+					runId,
+					"--n-concurrent",
+					"1",
+					"--no-livestream",
+					"--cleanup"
+				],
+				async parseReport(dir) {
+					const report = await readJsonReport(join(dir, "runs", runId, "results.json"));
+					const resolved = (report.resolved_ids ?? []).includes(task.id);
+					const trial = report.results?.find((r) => r.task_id === task.id);
+					return {
+						resolved,
+						score: resolved ? 1 : 0,
+						detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {})
+					};
+				}
+			});
+		}
+	};
+}
+//#endregion
+export { createTerminalBenchAdapter };
+
 //# sourceMappingURL=terminal-bench.js.map

package/dist/benchmarks/terminal-bench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"terminal-bench.js","names":[],"sources":["../../src/benchmarks/terminal-bench.ts"],"sourcesContent":["/**\n * Terminal-Bench adapter. Each task = a Docker environment + an English\n * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact\n * is the shell script the agent ran to attempt the task; the judge REPLAYS that\n * script in a fresh task container via the Terminal-Bench harness (`tb run` with\n * our ScriptAgent), then the task's own verifier scores the resulting state.\n * Fully deterministic — no LLM judge, no self-authored score.\n *\n * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`\n * installed + a running Docker daemon (per-task images are built on first run).\n * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the\n * Terminal-Bench registry on first run.\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay\n * argv, and the results.json shape.\n */\n\nimport { join } from 'node:path'\nimport {\n  benchRoot,\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  venvBinAt,\n  venvPythonAt,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.\n// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can\n// override the env without reloading this module.\nconst terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'\nconst terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())\nconst terminalBenchBin = (): string => venvBinAt(terminalBenchVenvDir(), 'tb')\n\n// Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the\n// installed CLI) and is the published launch task set. name==version is what `tb\n// run -d` and `tb datasets download -d` both accept.\nconst DATASET = 'terminal-bench-core'\nconst DATASET_VERSION = '0.1.1'\nconst DATASET_REF = `${DATASET}==${DATASET_VERSION}`\n\n// Bundled fixture: when no ids/limit are given, load these. hello-world is the\n// fastest deterministic task (prebuilt python image, file-write verifier) so the\n// adapter is runnable without a large pull.\nconst FIXTURE_IDS = ['hello-world']\n\n// Import path the harness uses to load our replay agent (cwd = benchRoot).\nconst SCRIPT_AGENT = 'tb_agents.script_agent:ScriptAgent'\n\ninterface TbTaskRow {\n  id: string\n  instruction: string\n  task_dir: string\n  solution: string | null\n}\n\ninterface TbReport {\n  resolved_ids?: string[]\n  results?: Array<{ task_id: string; is_resolved: boolean | null; parser_results?: unknown }>\n}\n\n/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on\n *  first run). Reads instruction from each task.yaml and the gold solution. */\nasync function loadRows(opts: LoadOptions): Promise<TbTaskRow[]> {\n  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS)\n  const limit = opts.limit ?? null\n  const script = `\nimport json, sys\nfrom pathlib import Path\nfrom terminal_bench.dataset.dataset import Dataset\nfrom terminal_bench.handlers.trial_handler import TaskPaths\n\nreq_ids = json.loads(sys.argv[1]) if sys.argv[1] else None\nlimit = json.loads(sys.argv[2]) if sys.argv[2] else None\n\nds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)\n\nimport yaml\nout = []\nfor task_dir in ds:\n    tp = TaskPaths(task_dir)\n    cfg = yaml.safe_load(tp.task_config_path.read_text())\n    try:\n        sol = tp.solution_path.read_text()\n    except FileNotFoundError:\n        sol = None\n    out.append({\n        \"id\": task_dir.name,\n        \"instruction\": cfg[\"instruction\"],\n        \"task_dir\": str(task_dir),\n        \"solution\": sol if (tp.solution_path.suffix == \".sh\" if sol is not None else False) else None,\n    })\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(\n    script,\n    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],\n    0,\n    terminalBenchPython(),\n  )\n  return JSON.parse(stdout) as TbTaskRow[]\n}\n\nexport function createTerminalBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'terminal-bench',\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['terminal_bench'],\n        requireDocker: true,\n        python: terminalBenchPython(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +\n          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +\n          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +\n          `override the dir with TERMINAL_BENCH_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +\n          `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const rows = await loadRows(opts)\n      if (rows.length === 0) {\n        throw new Error(\n          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} ` +\n            `(dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`,\n        )\n      }\n      return rows.map(\n        (r): BenchTask => ({\n          id: r.id,\n          split: DATASET_VERSION,\n          prompt: r.instruction,\n          metadata: {\n            dataset: DATASET,\n            datasetVersion: DATASET_VERSION,\n            datasetRef: DATASET_REF,\n            taskDir: r.task_dir,\n            solution: r.solution,\n            instruction: r.instruction,\n          },\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's solution.sh (the oracle script). solution.yaml tasks have\n      // no shell-script artifact form here, so they return undefined (cannot be\n      // verify-judged via the script-replay seam — use a .sh-solution task).\n      const sol = task.metadata?.solution\n      return typeof sol === 'string' ? sol : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', `${task.id}-${Date.now()}`)\n      return runStagedJudge({\n        tmpPrefix: 'tbench-',\n        bin: terminalBenchBin(),\n        cwd: () => benchRoot,\n        async stage(dir) {\n          await stageFile(join(dir, 'attempt.sh'), artifact)\n        },\n        // The harness builds a fresh task container, runs ScriptAgent (which replays\n        // the artifact script), then runs the task's verifier. --no-livestream keeps\n        // stdout sane; --cleanup removes the per-run images.\n        argv: (dir) => [\n          'run',\n          '-d', DATASET_REF,\n          '-t', task.id,\n          '--agent-import-path', SCRIPT_AGENT,\n          '--agent-kwarg', `script_path=${join(dir, 'attempt.sh')}`,\n          '--output-path', join(dir, 'runs'),\n          '--run-id', runId,\n          '--n-concurrent', '1',\n          '--no-livestream',\n          '--cleanup',\n        ],\n        async parseReport(dir) {\n          const report = await readJsonReport<TbReport>(join(dir, 'runs', runId, 'results.json'))\n          const resolved = (report.resolved_ids ?? []).includes(task.id)\n          const trial = report.results?.find((r) => r.task_id === task.id)\n          return {\n            resolved,\n            score: resolved ? 1 : 0,\n            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {}),\n          }\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAmCA,MAAM,6BAAqC,QAAQ,IAAI,uBAAuB;AAC9E,MAAM,4BAAoC,aAAa,qBAAqB,CAAC;AAC7E,MAAM,yBAAiC,UAAU,qBAAqB,GAAG,IAAI;AAK7E,MAAM,UAAU;AAChB,MAAM,kBAAkB;AACxB,MAAM,cAAc,GAAG,QAAQ,IAAI;AAKnC,MAAM,cAAc,CAAC,aAAa;AAGlC,MAAM,eAAe;;;AAgBrB,eAAe,SAAS,MAAyC;CAC/D,MAAM,MAAM,KAAK,QAAQ,KAAK,QAAQ,OAAO;CAC7C,MAAM,QAAQ,KAAK,SAAS;CA6B5B,MAAM,SAAS,MAAM,cACnB;;;;;;;;;oBApBgB,KAAK,UAAU,OAAO,EAAE,YAAY,KAAK,UAAU,eAAe,EAAE;;;;;;;;;;;;;;;;;;GAqBpF,CAAC,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,UAAU,OAAO,OAAO,KAAK,IAAI,EAAE,GACpE,GACA,oBAAoB,CACtB;CACA,OAAO,KAAK,MAAM,MAAM;AAC1B;AAEA,SAAgB,6BAA+C;CAC7D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,gBAAgB;IAC1B,eAAe;IACf,QAAQ,oBAAoB;IAC5B,KACE,kCAAkC,qBAAqB,EAAE,YAChD,qBAAqB,EAAE,0QAIzB,YAAY;GACvB,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,OAAO,MAAM,SAAS,IAAI;GAChC,IAAI,KAAK,WAAW,GAClB,MAAM,IAAI,MACR,kDAAkD,KAAK,UAAU,IAAI,EAAE,YACzD,YAAY,iDAC5B;GAEF,OAAO,KAAK,KACT,OAAkB;IACjB,IAAI,EAAE;IACN,OAAO;IACP,QAAQ,EAAE;IACV,UAAU;KACR,SAAS;KACT,gBAAgB;KAChB,YAAY;KACZ,SAAS,EAAE;KACX,UAAU,EAAE;KACZ,aAAa,EAAE;IACjB;GACF,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,MAAM,KAAK,UAAU;GAC3B,OAAO,OAAO,QAAQ,WAAW,MAAM,KAAA;EACzC;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,QAAQ,UAAU,SAAS,GAAG,KAAK,GAAG,GAAG,KAAK,IAAI,GAAG;GAC3D,OAAO,eAAe;IACpB,WAAW;IACX,KAAK,iBAAiB;IACtB,WAAW;IACX,MAAM,MAAM,KAAK;KACf,MAAM,UAAU,KAAK,KAAK,YAAY,GAAG,QAAQ;IACnD;IAIA,OAAO,QAAQ;KACb;KACA;KAAM;KACN;KAAM,KAAK;KACX;KAAuB;KACvB;KAAiB,eAAe,KAAK,KAAK,YAAY;KACtD;KAAiB,KAAK,KAAK,MAAM;KACjC;KAAY;KACZ;KAAkB;KAClB;KACA;IACF;IACA,MAAM,YAAY,KAAK;KACrB,MAAM,SAAS,MAAM,eAAyB,KAAK,KAAK,QAAQ,OAAO,cAAc,CAAC;KACtF,MAAM,YAAY,OAAO,gBAAgB,CAAC,EAAA,CAAG,SAAS,KAAK,EAAE;KAC7D,MAAM,QAAQ,OAAO,SAAS,MAAM,MAAM,EAAE,YAAY,KAAK,EAAE;KAC/D,OAAO;MACL;MACA,OAAO,WAAW,IAAI;MACtB,QAAQ,KAAK,UAAU,OAAO,kBAAkB,OAAO,gBAAgB,CAAC,CAAC;KAC3E;IACF;GACF,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/toollm.d.ts

@@ -1,16 +1,8 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * ToolLLM/ToolBench adapter.
- *
- * ToolBench task loading is useful for breadth, but the official ToolEval pass
- * rate evaluator is LLM-based and stochastic. This adapter therefore scores
- * only ToolBench's deterministic API-selection labels (`relevant APIs`). It
- * never records a full ToolEval pass-rate score.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/toollm.d.ts
 declare const toollmOutput: OutputAdapter<string>;
 declare function createToolLlmAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createToolLlmAdapter, toollmOutput };
+//# sourceMappingURL=toollm.d.ts.map