@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/benchmarks/tau-bench-shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau2-bench.d.ts

@@ -0,0 +1,7 @@
+import * as _tangle_network_agent_runtime_loops from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+declare const tau2ResultsOutput: _tangle_network_agent_runtime_loops.OutputAdapter<string>;
+declare function createTau2BenchAdapter(): BenchmarkAdapter;
+
+export { createTau2BenchAdapter, tau2ResultsOutput };

package/dist/benchmarks/tau2-bench.js

@@ -0,0 +1,11 @@
+import {
+  createTau2BenchAdapter,
+  tau2ResultsOutput
+} from "../chunk-HBSWHQNJ.js";
+import "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTau2BenchAdapter,
+  tau2ResultsOutput
+};
+//# sourceMappingURL=tau2-bench.js.map

package/dist/benchmarks/tau2-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau3-banking.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * tau3-banking adapter.
+ *
+ * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with
+ * package namespace `tau2` and a new `banking_knowledge` domain. This adapter
+ * reuses the shared tau reward recomputation path and only changes the default
+ * domain/env names.
+ */
+
+declare function createTau3BankingAdapter(): BenchmarkAdapter;
+
+export { createTau3BankingAdapter };

package/dist/benchmarks/tau3-banking.js

@@ -0,0 +1,9 @@
+import {
+  createTau3BankingAdapter
+} from "../chunk-UAIOHCUK.js";
+import "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTau3BankingAdapter
+};
+//# sourceMappingURL=tau3-banking.js.map

package/dist/benchmarks/tau3-banking.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/terminal-bench.d.ts

@@ -0,0 +1,24 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Terminal-Bench adapter. Each task = a Docker environment + an English
+ * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact
+ * is the shell script the agent ran to attempt the task; the judge REPLAYS that
+ * script in a fresh task container via the Terminal-Bench harness (`tb run` with
+ * our ScriptAgent), then the task's own verifier scores the resulting state.
+ * Fully deterministic — no LLM judge, no self-authored score.
+ *
+ * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`
+ * installed + a running Docker daemon (per-task images are built on first run).
+ * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the
+ * Terminal-Bench registry on first run.
+ *
+ * Process/Docker/report plumbing is shared via ./_harness; this file owns the
+ * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay
+ * argv, and the results.json shape.
+ */
+
+declare function createTerminalBenchAdapter(): BenchmarkAdapter;
+
+export { createTerminalBenchAdapter };

package/dist/benchmarks/terminal-bench.js

@@ -0,0 +1,8 @@
+import {
+  createTerminalBenchAdapter
+} from "../chunk-JRWWGMK7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTerminalBenchAdapter
+};
+//# sourceMappingURL=terminal-bench.js.map

package/dist/benchmarks/terminal-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/toollm.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * ToolLLM/ToolBench adapter.
+ *
+ * ToolBench task loading is useful for breadth, but the official ToolEval pass
+ * rate evaluator is LLM-based and stochastic. This adapter therefore scores
+ * only ToolBench's deterministic API-selection labels (`relevant APIs`). It
+ * never records a full ToolEval pass-rate score.
+ */
+
+declare const toollmOutput: OutputAdapter<string>;
+declare function createToolLlmAdapter(): BenchmarkAdapter;
+
+export { createToolLlmAdapter, toollmOutput };

package/dist/benchmarks/toollm.js

@@ -0,0 +1,10 @@
+import {
+  createToolLlmAdapter,
+  toollmOutput
+} from "../chunk-2PVVP7GN.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createToolLlmAdapter,
+  toollmOutput
+};
+//# sourceMappingURL=toollm.js.map

package/dist/benchmarks/toollm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/trata-hedge.d.ts

@@ -0,0 +1,32 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial
+ * analysis tasks across 6 domains (private equity, managed care, industrials,
+ * vertical SaaS, REITs, insurance). Each task bundles real earnings-call
+ * transcripts, financial statements, press releases, and SEC filings under
+ * `environment/data/`; the agent must produce a grounded analysis citing those
+ * files.
+ *
+ * Harbor architecture: the original benchmark runs each task in a Docker
+ * container with file-read tools. This adapter skips Harbor and embeds the data
+ * files directly in the worker prompt so any router backend can score it. Large
+ * files (> 30 KB) are truncated to fit within model context limits. The judge
+ * reads the actual files from disk for citation verification.
+ *
+ * Judge: a 3-stage LLM cascade faithful to grade.py —
+ *   Task 1: hallucination check (cited-file context + agent answer)
+ *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)
+ *   Task 3: synthesis check (ground truth + agent answer)
+ * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,
+ * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;
+ * resolved = score 4 (sparse reward).
+ *
+ * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.
+ * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench
+ */
+
+declare function createTrataHedgeAdapter(): BenchmarkAdapter;
+
+export { createTrataHedgeAdapter };

package/dist/benchmarks/trata-hedge.js

@@ -0,0 +1,7 @@
+import {
+  createTrataHedgeAdapter
+} from "../chunk-VQRS7VUC.js";
+export {
+  createTrataHedgeAdapter
+};
+//# sourceMappingURL=trata-hedge.js.map

package/dist/benchmarks/trata-hedge.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/types.d.ts

@@ -0,0 +1,107 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * BenchmarkAdapter — the seam every external benchmark implements so the
+ * agent-runtime loop can be A/B'd (blind vs steering vs steering+memory) against
+ * a REAL, DETERMINISTIC judge. The worker produces an `artifact` (a patch, a
+ * command transcript, a final state); the benchmark's own harness scores it.
+ *
+ * The point: no self-authored judge, no invented score noise. The number comes
+ * from the benchmark's published evaluation harness.
+ */
+
+interface BenchTask {
+    /** Stable benchmark instance id. */
+    id: string;
+    /** The task statement handed to the worker agent. */
+    prompt: string;
+    split?: string;
+    /** Benchmark-specific fields the worker/judge need (repo, base_commit, gold, …). */
+    metadata?: Record<string, unknown>;
+}
+interface JudgeArtifactFileReceipt {
+    /** POSIX path relative to the capture directory. */
+    path: string;
+    /** Exact byte length of the retained file or symbolic-link target. */
+    byteLength: number;
+    /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */
+    sha256: `sha256:${string}`;
+    kind: 'file' | 'symlink';
+}
+/** Durable evidence written before a staged evaluator's temporary directory is removed. */
+interface JudgeArtifactReceipt {
+    schema: 'agent-bench/judge-artifacts/v1';
+    /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */
+    directory: string;
+    /** Exact copy of the evaluator working directory. */
+    evaluatorDirectory: string;
+    manifestPath: string;
+    evaluatorSucceeded: boolean;
+    files: JudgeArtifactFileReceipt[];
+    fileCount: number;
+    byteLength: number;
+    /** SHA-256 over every sorted path, kind, byte length, and content hash. */
+    treeSha256: `sha256:${string}`;
+}
+interface BenchScore {
+    /** Did the deterministic judge pass (tests resolved / state correct)? */
+    resolved: boolean;
+    /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
+    score: number;
+    detail?: string;
+    /** Present only when the caller explicitly requested durable judge evidence. */
+    judgeArtifacts?: JudgeArtifactReceipt;
+}
+interface LoadOptions {
+    limit?: number;
+    split?: string;
+    ids?: string[];
+}
+interface BenchmarkAdapter {
+    readonly name: string;
+    /** Throw with actionable guidance when the harness/judge isn't installed/runnable. */
+    preflight(): Promise<void>;
+    loadTasks(opts?: LoadOptions): Promise<BenchTask[]>;
+    /** DETERMINISTIC judge: score the worker's produced artifact for a task. */
+    judge(task: BenchTask, artifact: string): Promise<BenchScore>;
+    /** Gold/oracle artifact — lets us self-verify the judge before spending model tokens. */
+    goldArtifact(task: BenchTask): Promise<string | undefined>;
+    /** How to extract the judged artifact from a run's event stream. Optional —
+     *  defaults to the agent's final answer text (the research/QA case). SWE sets
+     *  it to a patch parser. This is `benchmark = adapter` owning its deliverable,
+     *  so the gate runner (`runGate` / `runBenchmark`) needs no
+     *  per-benchmark branching. */
+    output?: OutputAdapter<string>;
+    /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream.
+     *  When set, the shot runner execs `command` in the STILL-ALIVE box after the agent
+     *  turn drains and uses its stdout as the judged artifact — the durable way to capture
+     *  a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent /
+     *  OpenHands read the diff from repo STATE), instead of hoping the model printed a
+     *  fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the
+     *  event-stream parse). `cwd` defaults to the box root. */
+    boxExtract?(task: BenchTask): {
+        command: string;
+        cwd?: string;
+    };
+    /** Optional workspace pre-stage run in the box BEFORE the agent shot (same
+     *  session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones
+     *  the instance repo at `base_commit` into a fixed path so the agent only edits
+     *  — the harness owns the checkout, not the (stochastic) model. A non-zero exit
+     *  fails the shot loud rather than letting the agent run against an empty box. */
+    boxSetup?(task: BenchTask): {
+        command: string;
+        cwd?: string;
+    };
+    /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
+     *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
+     *  not as a chat completion) — the experiment uses this instead of the
+     *  BACKEND-selected client; the steer still flows through the per-round prompt.
+     *  Typed loosely to avoid a runtime import cycle; the harness casts it. */
+    leafClient?: (cfg: {
+        model: string;
+        routerBaseUrl: string;
+        routerKey: string;
+    }) => unknown;
+}
+
+export type { BenchScore, BenchTask, BenchmarkAdapter, JudgeArtifactFileReceipt, JudgeArtifactReceipt, LoadOptions };

package/dist/benchmarks/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/benchmarks/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/webarena-verified.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * WebArena-Verified adapter (ServiceNow/webarena-verified).
+ *
+ * Worker artifact = a WebArena-Verified run output directory, not final chat text.
+ * Judge = the official `webarena_verified eval-tasks` evaluator over that output
+ * directory. The adapter refuses to score a plain answer so we do not turn a DOM
+ * benchmark into a fake text benchmark.
+ */
+
+declare const webarenaOutputDirOutput: OutputAdapter<string>;
+declare function createWebArenaVerifiedAdapter(): BenchmarkAdapter;
+
+export { createWebArenaVerifiedAdapter, webarenaOutputDirOutput };

package/dist/benchmarks/webarena-verified.js

@@ -0,0 +1,10 @@
+import {
+  createWebArenaVerifiedAdapter,
+  webarenaOutputDirOutput
+} from "../chunk-Z7ML6L77.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createWebArenaVerifiedAdapter,
+  webarenaOutputDirOutput
+};
+//# sourceMappingURL=webarena-verified.js.map

package/dist/benchmarks/webarena-verified.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/chunk-2PVVP7GN.js

@@ -0,0 +1,197 @@
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/toollm.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "toollm.json");
+var DEFAULT_QUERY_REL = join("data_example", "instruction", "G1_query.json");
+var toolbenchDir = () => process.env.TOOLBENCH_DIR;
+var queryFile = (dir) => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL);
+var toollmOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    return text.trim();
+  }
+};
+function rowToTask(row) {
+  const relevantApis = normalizeApiPairs(row["relevant APIs"] ?? []);
+  const meta = {
+    queryId: String(row.query_id),
+    apiList: row.api_list ?? [],
+    relevantApis,
+    deterministicJudge: "api-selection"
+  };
+  return {
+    id: String(row.query_id),
+    prompt: [
+      "Solve this ToolLLM/ToolBench API-use task.",
+      "Use only the listed APIs/tools and return the completed tool-use trace plus final answer.",
+      "",
+      `Query: ${row.query}`,
+      "",
+      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
+      "",
+      'Return the APIs you used as JSON: {"api_calls":[{"tool_name":"...","api_name":"..."}]}.'
+    ].join("\n"),
+    metadata: meta
+  };
+}
+function normalizeApiPart(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+function apiKey(pair) {
+  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`;
+}
+function normalizeApiPairs(value) {
+  if (!Array.isArray(value)) return [];
+  const out = [];
+  for (const item of value) {
+    if (Array.isArray(item) && typeof item[0] === "string" && typeof item[1] === "string") {
+      out.push([item[0], item[1]]);
+    } else if (item && typeof item === "object" && typeof item.tool_name === "string" && typeof item.api_name === "string") {
+      out.push([item.tool_name, item.api_name]);
+    }
+  }
+  return out;
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.relevantApis)) {
+    throw new Error(`ToolLLM task ${task.id} missing metadata \u2014 loadTasks did not populate deterministic API-selection labels`);
+  }
+  return md;
+}
+function selectRows(rows, opts) {
+  let tasks = rows.map(rowToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+function assertDeterministicSubset(tasks, source) {
+  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id);
+  if (missing.length > 0) {
+    throw new Error(
+      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(", ")}. Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.`
+    );
+  }
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[toollm] TOOLLM_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  const tasks = selectRows(rows, opts);
+  assertDeterministicSubset(tasks, FIXTURES);
+  return tasks;
+}
+async function loadOfficialTasks(dir, opts) {
+  const source = queryFile(dir);
+  const tasks = selectRows(JSON.parse(await readFile(source, "utf8")), opts);
+  assertDeterministicSubset(tasks, source);
+  return tasks;
+}
+function extractJsonBlock(text) {
+  const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
+  const raw = (fences.at(-1)?.[1] ?? text).trim();
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+}
+function extractCalledApis(text, expected) {
+  const parsed = extractJsonBlock(text);
+  if (parsed && typeof parsed === "object") {
+    const raw = parsed;
+    const fromApiCalls = normalizeApiPairs(raw.api_calls);
+    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: "structured-json" };
+    const fromCalls = normalizeApiPairs(raw.calls);
+    if (fromCalls.length > 0) return { pairs: fromCalls, source: "structured-json" };
+  }
+  const lower = text.toLowerCase();
+  return {
+    pairs: expected.filter(([tool, api]) => {
+      const toolNeedle = normalizeApiPart(tool);
+      const apiNeedle = normalizeApiPart(api);
+      const compactText = lower.replace(/[^a-z0-9]+/g, "");
+      return compactText.includes(`${toolNeedle}${apiNeedle}`) || lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase());
+    }),
+    source: "text-mention"
+  };
+}
+function scoreApiSelection(task, artifact) {
+  const meta = readMeta(task);
+  if (meta.relevantApis.length === 0) {
+    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`);
+  }
+  const expected = new Set(meta.relevantApis.map(apiKey));
+  const extracted = extractCalledApis(artifact, meta.relevantApis);
+  const calledPairs = extracted.pairs;
+  const called = new Set(calledPairs.map(apiKey));
+  const truePositives = [...called].filter((key) => expected.has(key)).length;
+  const precision = called.size === 0 ? 0 : truePositives / called.size;
+  const recall = truePositives / expected.size;
+  const score = expected.size === 0 ? 0 : recall;
+  const resolved = extracted.source === "structured-json" && recall === 1 && precision === 1;
+  return {
+    resolved,
+    score,
+    detail: JSON.stringify({
+      scoring: "api-selection-only",
+      extractionSource: extracted.source,
+      queryId: meta.queryId,
+      expected: meta.relevantApis,
+      called: calledPairs,
+      precision,
+      recall,
+      fullToolEvalScore: null
+    })
+  };
+}
+function createToolLlmAdapter() {
+  const fixturesMode = process.env.TOOLLM_FIXTURES === "1";
+  return {
+    name: "toollm",
+    output: toollmOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = toolbenchDir();
+      if (!dir) {
+        throw new Error("TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.");
+      }
+      await loadOfficialTasks(dir, { limit: 1 });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const dir = toolbenchDir();
+      if (!dir) throw new Error("TOOLBENCH_DIR is required to load ToolLLM tasks");
+      return loadOfficialTasks(dir, opts);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      if (meta.relevantApis.length === 0) return void 0;
+      return JSON.stringify({
+        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name }))
+      }, null, 2);
+    },
+    async judge(task, artifact) {
+      return scoreApiSelection(task, artifact);
+    }
+  };
+}
+
+export {
+  toollmOutput,
+  createToolLlmAdapter
+};
+//# sourceMappingURL=chunk-2PVVP7GN.js.map

package/dist/chunk-2PVVP7GN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,IAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,IAAM,eAAe,MAA0B,QAAQ,IAAI;AAC3D,IAAM,YAAY,CAAC,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEhG,IAAM,eAAsC;AAAA,EACjD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,UAAU,KAA8B;AAC/C,QAAM,eAAe,kBAAkB,IAAI,eAAe,KAAK,CAAC,CAAC;AACjE,QAAM,OAAsB;AAAA,IAC1B,SAAS,OAAO,IAAI,QAAQ;AAAA,IAC5B,SAAS,IAAI,YAAY,CAAC;AAAA,IAC1B;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,QAAQ;AAAA,IACvB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,MACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,MAC9D;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;AAC/C,SAAO,MAAM,YAAY,EAAE,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;AACvD,SAAO,GAAG,iBAAiB,KAAK,CAAC,CAAC,CAAC,IAAI,iBAAiB,KAAK,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,kBAAkB,OAAyC;AAClE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAA+B,CAAC;AACtC,aAAW,QAAQ,OAAO;AACxB,QAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,CAAC,MAAM,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU;AACrF,UAAI,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7B,WACE,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UACxD;AACA,UAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;AAAA,IAC/F;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GAAG;AAC1C,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wFAAmF;AAAA,EAC5H;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;AACxE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;AACpF,QAAM,UAAU,MAAM,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,aAAa,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,EAAE;AACtG,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,0DAA0D,QAAQ,MAAM,IAAI,MAAM,MAAM,iBAAiB,MAAM,KAAK,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAEpJ;AAAA,EACF;AACF;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6CAAwC,KAAK,MAAM,mBAAmB;AACnF,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,4BAA0B,OAAO,QAAQ;AACzC,SAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,SAAS,UAAU,GAAG;AAC5B,QAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;AAC3F,4BAA0B,OAAO,MAAM;AACvC,SAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;AAC7F,QAAM,SAAS,iBAAiB,IAAI;AACpC,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,UAAM,eAAe,kBAAkB,IAAI,SAAS;AACpD,QAAI,aAAa,SAAS,EAAG,QAAO,EAAE,OAAO,cAAc,QAAQ,kBAAkB;AACrF,UAAM,YAAY,kBAAkB,IAAI,KAAK;AAC7C,QAAI,UAAU,SAAS,EAAG,QAAO,EAAE,OAAO,WAAW,QAAQ,kBAAkB;AAAA,EACjF;AAEA,QAAM,QAAQ,KAAK,YAAY;AAC/B,SAAO;AAAA,IACL,OAAO,SAAS,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM;AACtC,YAAM,aAAa,iBAAiB,IAAI;AACxC,YAAM,YAAY,iBAAiB,GAAG;AACtC,YAAM,cAAc,MAAM,QAAQ,eAAe,EAAE;AACnD,aAAO,YAAY,SAAS,GAAG,UAAU,GAAG,SAAS,EAAE,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;AAAA,IACrI,CAAC;AAAA,IACD,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;AACxE,QAAM,OAAO,SAAS,IAAI;AAC1B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,+DAA+D;AAAA,EACxG;AACA,QAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;AACtD,QAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;AAC/D,QAAM,cAAc,UAAU;AAC9B,QAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;AAC9C,QAAM,gBAAgB,CAAC,GAAG,MAAM,EAAE,OAAO,CAAC,QAAQ,SAAS,IAAI,GAAG,CAAC,EAAE;AACrE,QAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;AACjE,QAAM,SAAS,gBAAgB,SAAS;AACxC,QAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;AACxC,QAAM,WAAW,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;AACzF,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS;AAAA,MACT,kBAAkB,UAAU;AAAA,MAC5B,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,oBAAoB;AAErD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,sHAAsH;AAAA,MACxI;AACA,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,KAAK,aAAa,WAAW,EAAG,QAAO;AAC3C,aAAO,KAAK,UAAU;AAAA,QACpB,WAAW,KAAK,aAAa,IAAI,CAAC,CAAC,WAAW,QAAQ,OAAO,EAAE,WAAW,SAAS,EAAE;AAAA,MACvF,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,kBAAkB,MAAM,QAAQ;AAAA,IACzC;AAAA,EACF;AACF;","names":[]}