@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-CLIKAXKH.js

@@ -1,276 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/bfcl.ts
-import { readFile, stat } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "bfcl.json");
-var DEFAULT_CATEGORY = "BFCL_v4_simple_python";
-var bfclDir = () => process.env.BFCL_DIR;
-var bfclCategory = () => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY;
-var bfclOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text.trim();
-  }
-};
-async function assertPath(path, label) {
-  try {
-    await stat(path);
-  } catch (err) {
-    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function readJsonl(raw) {
-  return raw.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
-}
-function taskFile(dir, category) {
-  return process.env.BFCL_DATA_FILE ?? join(dir, "bfcl_eval", "data", `${category}.json`);
-}
-function answerFile(dir, category) {
-  return process.env.BFCL_ANSWER_FILE ?? join(dir, "bfcl_eval", "data", "possible_answer", `${category}.json`);
-}
-function questionText(question) {
-  if (typeof question === "string") return question;
-  if (!Array.isArray(question)) return JSON.stringify(question, null, 2);
-  const turns = [];
-  for (const item of question.flat(3)) {
-    if (item && typeof item === "object") {
-      const role = typeof item.role === "string" ? item.role : "user";
-      const content = item.content;
-      if (typeof content === "string") turns.push(`${role}: ${content}`);
-    }
-  }
-  return turns.length > 0 ? turns.join("\n") : JSON.stringify(question, null, 2);
-}
-function normalizeScalar(value) {
-  if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)));
-  if (typeof value === "string") return value.trim().toLowerCase();
-  return JSON.stringify(value);
-}
-function normalizeAllowed(value) {
-  return Array.isArray(value) ? value : [value];
-}
-function groundTruthToCalls(value) {
-  if (!Array.isArray(value)) return [];
-  const out = [];
-  for (const item of value) {
-    if (!item || typeof item !== "object") continue;
-    for (const [name, args] of Object.entries(item)) {
-      if (!args || typeof args !== "object" || Array.isArray(args)) continue;
-      const normalized = {};
-      for (const [argName, allowed] of Object.entries(args)) {
-        normalized[argName] = normalizeAllowed(allowed);
-      }
-      out.push({ name, arguments: normalized });
-    }
-  }
-  return out;
-}
-function rowToTask(row, answer, category) {
-  const expected = groundTruthToCalls(answer.ground_truth);
-  if (expected.length === 0) {
-    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`);
-  }
-  const meta = {
-    rowId: row.id,
-    category,
-    functions: row.function,
-    expected,
-    scoring: "bfcl-ground-truth-subset"
-  };
-  return {
-    id: row.id,
-    split: category,
-    prompt: [
-      "Solve this BFCL function-calling task.",
-      'Return only JSON: {"function_calls":[{"name":"...","arguments":{...}}]}.',
-      "",
-      questionText(row.question),
-      "",
-      `Available functions: ${JSON.stringify(row.function, null, 2)}`
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.expected)) {
-    throw new Error(`BFCL task ${task.id} missing expected calls \u2014 loadTasks did not populate metadata`);
-  }
-  return md;
-}
-function selectTasks(tasks, opts) {
-  let out = tasks;
-  if (opts.split) out = out.filter((task) => task.split === opts.split);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    out = out.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    out = out.slice(0, opts.limit);
-  }
-  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`);
-  return out;
-}
-function buildTasks(rows, answers, category, opts) {
-  const byId = new Map(answers.map((answer) => [answer.id, answer]));
-  const tasks = rows.map((row) => {
-    const answer = byId.get(row.id);
-    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`);
-    return rowToTask(row, answer, category);
-  });
-  return selectTasks(tasks, opts);
-}
-async function loadFixtures(opts) {
-  const fixture = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[bfcl] BFCL_FIXTURES=1 \u2014 loading ${fixture.rows.length} adapter fixtures`);
-  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts);
-}
-async function loadOfficialTasks(dir, opts) {
-  const category = bfclCategory();
-  const dataPath = taskFile(dir, category);
-  const answersPath = answerFile(dir, category);
-  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, "utf8"), readFile(answersPath, "utf8")]);
-  const rows = readJsonl(rawRows);
-  const answers = readJsonl(rawAnswers);
-  return buildTasks(rows, answers, category, opts);
-}
-function extractJsonBlock(text) {
-  const fences = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)];
-  const raw = (fences.at(-1)?.[1] ?? text).trim();
-  try {
-    return JSON.parse(raw);
-  } catch {
-    return void 0;
-  }
-}
-function normalizeArguments(value) {
-  if (typeof value === "string") {
-    try {
-      return normalizeArguments(JSON.parse(value));
-    } catch {
-      return {};
-    }
-  }
-  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
-  return value;
-}
-function callFromObject(value) {
-  if (!value || typeof value !== "object") return void 0;
-  const raw = value;
-  if (raw.function && typeof raw.function === "object") {
-    const fn = raw.function;
-    if (typeof fn.name === "string") return { name: fn.name, arguments: normalizeArguments(fn.arguments) };
-  }
-  const name = typeof raw.name === "string" ? raw.name : typeof raw.function_name === "string" ? raw.function_name : typeof raw.tool_name === "string" ? raw.tool_name : void 0;
-  if (!name) return void 0;
-  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) };
-}
-function extractActualCalls(artifact) {
-  const parsed = extractJsonBlock(artifact);
-  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call) => Boolean(call));
-  if (parsed && typeof parsed === "object") {
-    const raw = parsed;
-    for (const key of ["function_calls", "tool_calls", "calls"]) {
-      if (Array.isArray(raw[key])) {
-        return raw[key].map(callFromObject).filter((call) => Boolean(call));
-      }
-    }
-    const single = callFromObject(raw);
-    if (single) return [single];
-  }
-  return [];
-}
-function argMatches(expectedValues, actual) {
-  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual));
-}
-function callMatches(expected, actual) {
-  if (expected.name !== actual.name) return false;
-  for (const [argName, allowed] of Object.entries(expected.arguments)) {
-    if (!(argName in actual.arguments)) {
-      if (allowed.some((value) => value === "" || value === null || value === void 0)) continue;
-      return false;
-    }
-    if (!argMatches(allowed, actual.arguments[argName])) return false;
-  }
-  return true;
-}
-function scoreCalls(task, artifact) {
-  const meta = readMeta(task);
-  const actual = extractActualCalls(artifact);
-  const used = /* @__PURE__ */ new Set();
-  let matched = 0;
-  for (const expected of meta.expected) {
-    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call));
-    if (index >= 0) {
-      used.add(index);
-      matched += 1;
-    }
-  }
-  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length;
-  const precision = actual.length === 0 ? 0 : matched / actual.length;
-  const score = recall;
-  return {
-    resolved: recall === 1 && precision === 1,
-    score,
-    detail: JSON.stringify({
-      scoring: meta.scoring,
-      category: meta.category,
-      expected: meta.expected,
-      actual,
-      precision,
-      recall,
-      fullBfclLeaderboardScore: null
-    })
-  };
-}
-function createBfclAdapter() {
-  const fixturesMode = process.env.BFCL_FIXTURES === "1";
-  return {
-    name: "bfcl",
-    output: bfclOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = bfclDir();
-      if (!dir) {
-        throw new Error(
-          "BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python."
-        );
-      }
-      const category = bfclCategory();
-      await assertPath(taskFile(dir, category), "BFCL task JSONL");
-      await assertPath(answerFile(dir, category), "BFCL possible_answer JSONL");
-      await loadOfficialTasks(dir, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = bfclDir();
-      if (!dir) throw new Error("BFCL_DIR is required to load official BFCL rows");
-      return loadOfficialTasks(dir, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return JSON.stringify({
-        function_calls: meta.expected.map((call) => ({
-          name: call.name,
-          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]]))
-        }))
-      }, null, 2);
-    },
-    async judge(task, artifact) {
-      return scoreCalls(task, artifact);
-    }
-  };
-}
-
-export {
-  bfclOutput,
-  createBfclAdapter
-};
-//# sourceMappingURL=chunk-CLIKAXKH.js.map

package/dist/chunk-CLIKAXKH.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/bfcl.ts"],"sourcesContent":["/**\n * Berkeley Function Calling Leaderboard adapter.\n *\n * Scope: deterministic function-call ground-truth categories from the official\n * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic\n * web-search/memory categories and BFCL's own model-response harness remain\n * upstream responsibilities. The adapter loads official JSONL rows plus their\n * `possible_answer` file and scores structured function-call artifacts against\n * allowed function/argument values.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'bfcl.json')\nconst DEFAULT_CATEGORY = 'BFCL_v4_simple_python'\n\ninterface BfclRow {\n  id: string\n  question: unknown\n  function: unknown\n}\n\ninterface BfclAnswerRow {\n  id: string\n  ground_truth: unknown\n}\n\ninterface BfclAllowedCall {\n  name: string\n  arguments: Record<string, unknown[]>\n}\n\ninterface BfclMeta {\n  rowId: string\n  category: string\n  functions: unknown\n  expected: BfclAllowedCall[]\n  scoring: 'bfcl-ground-truth-subset'\n}\n\ninterface BfclActualCall {\n  name: string\n  arguments: Record<string, unknown>\n}\n\nconst bfclDir = (): string | undefined => process.env.BFCL_DIR\nconst bfclCategory = (): string => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY\n\nexport const bfclOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nasync function assertPath(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readJsonl(raw: string): unknown[] {\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nfunction taskFile(dir: string, category: string): string {\n  return process.env.BFCL_DATA_FILE ?? join(dir, 'bfcl_eval', 'data', `${category}.json`)\n}\n\nfunction answerFile(dir: string, category: string): string {\n  return process.env.BFCL_ANSWER_FILE ?? join(dir, 'bfcl_eval', 'data', 'possible_answer', `${category}.json`)\n}\n\nfunction questionText(question: unknown): string {\n  if (typeof question === 'string') return question\n  if (!Array.isArray(question)) return JSON.stringify(question, null, 2)\n  const turns: string[] = []\n  for (const item of question.flat(3)) {\n    if (item && typeof item === 'object') {\n      const role = typeof (item as { role?: unknown }).role === 'string' ? (item as { role: string }).role : 'user'\n      const content = (item as { content?: unknown }).content\n      if (typeof content === 'string') turns.push(`${role}: ${content}`)\n    }\n  }\n  return turns.length > 0 ? turns.join('\\n') : JSON.stringify(question, null, 2)\n}\n\nfunction normalizeScalar(value: unknown): string {\n  if (typeof value === 'number') return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)))\n  if (typeof value === 'string') return value.trim().toLowerCase()\n  return JSON.stringify(value)\n}\n\nfunction normalizeAllowed(value: unknown): unknown[] {\n  return Array.isArray(value) ? value : [value]\n}\n\nfunction groundTruthToCalls(value: unknown): BfclAllowedCall[] {\n  if (!Array.isArray(value)) return []\n  const out: BfclAllowedCall[] = []\n  for (const item of value) {\n    if (!item || typeof item !== 'object') continue\n    for (const [name, args] of Object.entries(item as Record<string, unknown>)) {\n      if (!args || typeof args !== 'object' || Array.isArray(args)) continue\n      const normalized: Record<string, unknown[]> = {}\n      for (const [argName, allowed] of Object.entries(args as Record<string, unknown>)) {\n        normalized[argName] = normalizeAllowed(allowed)\n      }\n      out.push({ name, arguments: normalized })\n    }\n  }\n  return out\n}\n\nfunction rowToTask(row: BfclRow, answer: BfclAnswerRow, category: string): BenchTask {\n  const expected = groundTruthToCalls(answer.ground_truth)\n  if (expected.length === 0) {\n    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`)\n  }\n  const meta: BfclMeta = {\n    rowId: row.id,\n    category,\n    functions: row.function,\n    expected,\n    scoring: 'bfcl-ground-truth-subset',\n  }\n  return {\n    id: row.id,\n    split: category,\n    prompt: [\n      'Solve this BFCL function-calling task.',\n      'Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.',\n      '',\n      questionText(row.question),\n      '',\n      `Available functions: ${JSON.stringify(row.function, null, 2)}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): BfclMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.expected)) {\n    throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`)\n  }\n  return md as unknown as BfclMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  let out = tasks\n  if (opts.split) out = out.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    out = out.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    out = out.slice(0, opts.limit)\n  }\n  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`)\n  return out\n}\n\nfunction buildTasks(rows: BfclRow[], answers: BfclAnswerRow[], category: string, opts: LoadOptions): BenchTask[] {\n  const byId = new Map(answers.map((answer) => [answer.id, answer]))\n  const tasks = rows.map((row) => {\n    const answer = byId.get(row.id)\n    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`)\n    return rowToTask(row, answer, category)\n  })\n  return selectTasks(tasks, opts)\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixture = JSON.parse(await readFile(FIXTURES, 'utf8')) as { rows: BfclRow[]; answers: BfclAnswerRow[]; category: string }\n  console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`)\n  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const category = bfclCategory()\n  const dataPath = taskFile(dir, category)\n  const answersPath = answerFile(dir, category)\n  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, 'utf8'), readFile(answersPath, 'utf8')])\n  const rows = readJsonl(rawRows) as BfclRow[]\n  const answers = readJsonl(rawAnswers) as BfclAnswerRow[]\n  return buildTasks(rows, answers, category, opts)\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\nfunction normalizeArguments(value: unknown): Record<string, unknown> {\n  if (typeof value === 'string') {\n    try {\n      return normalizeArguments(JSON.parse(value))\n    } catch {\n      return {}\n    }\n  }\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return {}\n  return value as Record<string, unknown>\n}\n\nfunction callFromObject(value: unknown): BfclActualCall | undefined {\n  if (!value || typeof value !== 'object') return undefined\n  const raw = value as Record<string, unknown>\n  if (raw.function && typeof raw.function === 'object') {\n    const fn = raw.function as Record<string, unknown>\n    if (typeof fn.name === 'string') return { name: fn.name, arguments: normalizeArguments(fn.arguments) }\n  }\n  const name =\n    typeof raw.name === 'string' ? raw.name\n      : typeof raw.function_name === 'string' ? raw.function_name\n        : typeof raw.tool_name === 'string' ? raw.tool_name\n          : undefined\n  if (!name) return undefined\n  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) }\n}\n\nfunction extractActualCalls(artifact: string): BfclActualCall[] {\n  const parsed = extractJsonBlock(artifact)\n  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    for (const key of ['function_calls', 'tool_calls', 'calls']) {\n      if (Array.isArray(raw[key])) {\n        return raw[key].map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n      }\n    }\n    const single = callFromObject(raw)\n    if (single) return [single]\n  }\n  return []\n}\n\nfunction argMatches(expectedValues: unknown[], actual: unknown): boolean {\n  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual))\n}\n\nfunction callMatches(expected: BfclAllowedCall, actual: BfclActualCall): boolean {\n  if (expected.name !== actual.name) return false\n  for (const [argName, allowed] of Object.entries(expected.arguments)) {\n    if (!(argName in actual.arguments)) {\n      if (allowed.some((value) => value === '' || value === null || value === undefined)) continue\n      return false\n    }\n    if (!argMatches(allowed, actual.arguments[argName])) return false\n  }\n  return true\n}\n\nfunction scoreCalls(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  const actual = extractActualCalls(artifact)\n  const used = new Set<number>()\n  let matched = 0\n  for (const expected of meta.expected) {\n    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call))\n    if (index >= 0) {\n      used.add(index)\n      matched += 1\n    }\n  }\n  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length\n  const precision = actual.length === 0 ? 0 : matched / actual.length\n  const score = recall\n  return {\n    resolved: recall === 1 && precision === 1,\n    score,\n    detail: JSON.stringify({\n      scoring: meta.scoring,\n      category: meta.category,\n      expected: meta.expected,\n      actual,\n      precision,\n      recall,\n      fullBfclLeaderboardScore: null,\n    }),\n  }\n}\n\nexport function createBfclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.BFCL_FIXTURES === '1'\n\n  return {\n    name: 'bfcl',\n    output: bfclOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = bfclDir()\n      if (!dir) {\n        throw new Error(\n          'BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.',\n        )\n      }\n      const category = bfclCategory()\n      await assertPath(taskFile(dir, category), 'BFCL task JSONL')\n      await assertPath(answerFile(dir, category), 'BFCL possible_answer JSONL')\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = bfclDir()\n      if (!dir) throw new Error('BFCL_DIR is required to load official BFCL rows')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return JSON.stringify({\n        function_calls: meta.expected.map((call) => ({\n          name: call.name,\n          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]])),\n        })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreCalls(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AAWA,SAAS,UAAU,YAAY;AAC/B,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AACxD,IAAM,mBAAmB;AA+BzB,IAAM,UAAU,MAA0B,QAAQ,IAAI;AACtD,IAAM,eAAe,MAAc,QAAQ,IAAI,iBAAiB;AAEzD,IAAM,aAAoC;AAAA,EAC/C,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,iBAAiB,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACnG;AACF;AAEA,SAAS,UAAU,KAAwB;AACzC,SAAO,IACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAS,SAAS,KAAa,UAA0B;AACvD,SAAO,QAAQ,IAAI,kBAAkB,KAAK,KAAK,aAAa,QAAQ,GAAG,QAAQ,OAAO;AACxF;AAEA,SAAS,WAAW,KAAa,UAA0B;AACzD,SAAO,QAAQ,IAAI,oBAAoB,KAAK,KAAK,aAAa,QAAQ,mBAAmB,GAAG,QAAQ,OAAO;AAC7G;AAEA,SAAS,aAAa,UAA2B;AAC/C,MAAI,OAAO,aAAa,SAAU,QAAO;AACzC,MAAI,CAAC,MAAM,QAAQ,QAAQ,EAAG,QAAO,KAAK,UAAU,UAAU,MAAM,CAAC;AACrE,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,SAAS,KAAK,CAAC,GAAG;AACnC,QAAI,QAAQ,OAAO,SAAS,UAAU;AACpC,YAAM,OAAO,OAAQ,KAA4B,SAAS,WAAY,KAA0B,OAAO;AACvG,YAAM,UAAW,KAA+B;AAChD,UAAI,OAAO,YAAY,SAAU,OAAM,KAAK,GAAG,IAAI,KAAK,OAAO,EAAE;AAAA,IACnE;AAAA,EACF;AACA,SAAO,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,UAAU,UAAU,MAAM,CAAC;AAC/E;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,OAAO,UAAU,SAAU,QAAO,OAAO,UAAU,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;AAC/G,MAAI,OAAO,UAAU,SAAU,QAAO,MAAM,KAAK,EAAE,YAAY;AAC/D,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,iBAAiB,OAA2B;AACnD,SAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAC9C;AAEA,SAAS,mBAAmB,OAAmC;AAC7D,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAAyB,CAAC;AAChC,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,QAAQ,OAAO,SAAS,SAAU;AACvC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,IAA+B,GAAG;AAC1E,UAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,EAAG;AAC9D,YAAM,aAAwC,CAAC;AAC/C,iBAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,IAA+B,GAAG;AAChF,mBAAW,OAAO,IAAI,iBAAiB,OAAO;AAAA,MAChD;AACA,UAAI,KAAK,EAAE,MAAM,WAAW,WAAW,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,UAAU,KAAc,QAAuB,UAA6B;AACnF,QAAM,WAAW,mBAAmB,OAAO,YAAY;AACvD,MAAI,SAAS,WAAW,GAAG;AACzB,UAAM,IAAI,MAAM,QAAQ,IAAI,EAAE,2DAA2D;AAAA,EAC3F;AACA,QAAM,OAAiB;AAAA,IACrB,OAAO,IAAI;AAAA,IACX;AAAA,IACA,WAAW,IAAI;AAAA,IACf;AAAA,IACA,SAAS;AAAA,EACX;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO;AAAA,IACP,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,IAAI,QAAQ;AAAA,MACzB;AAAA,MACA,wBAAwB,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC,CAAC;AAAA,IAC/D,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,QAAQ,GAAG;AACtC,UAAM,IAAI,MAAM,aAAa,KAAK,EAAE,oEAA+D;AAAA,EACrG;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;AACvE,MAAI,MAAM;AACV,MAAI,KAAK,MAAO,OAAM,IAAI,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AACpE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,UAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAC9C,WAAW,KAAK,UAAU,QAAW;AACnC,UAAM,IAAI,MAAM,GAAG,KAAK,KAAK;AAAA,EAC/B;AACA,MAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,0BAA0B,KAAK,UAAU,IAAI,CAAC,EAAE;AACtF,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,SAA0B,UAAkB,MAAgC;AAC/G,QAAM,OAAO,IAAI,IAAI,QAAQ,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;AACjE,QAAM,QAAQ,KAAK,IAAI,CAAC,QAAQ;AAC9B,UAAM,SAAS,KAAK,IAAI,IAAI,EAAE;AAC9B,QAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,QAAQ,QAAQ,iCAAiC,IAAI,EAAE,EAAE;AACtF,WAAO,UAAU,KAAK,QAAQ,QAAQ;AAAA,EACxC,CAAC;AACD,SAAO,YAAY,OAAO,IAAI;AAChC;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC3D,UAAQ,KAAK,yCAAoC,QAAQ,KAAK,MAAM,mBAAmB;AACvF,SAAO,WAAW,QAAQ,MAAM,QAAQ,SAAS,QAAQ,UAAU,IAAI;AACzE;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,WAAW,aAAa;AAC9B,QAAM,WAAW,SAAS,KAAK,QAAQ;AACvC,QAAM,cAAc,WAAW,KAAK,QAAQ;AAC5C,QAAM,CAAC,SAAS,UAAU,IAAI,MAAM,QAAQ,IAAI,CAAC,SAAS,UAAU,MAAM,GAAG,SAAS,aAAa,MAAM,CAAC,CAAC;AAC3G,QAAM,OAAO,UAAU,OAAO;AAC9B,QAAM,UAAU,UAAU,UAAU;AACpC,SAAO,WAAW,MAAM,SAAS,UAAU,IAAI;AACjD;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,+BAA+B,CAAC;AACjE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI;AACF,aAAO,mBAAmB,KAAK,MAAM,KAAK,CAAC;AAAA,IAC7C,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACzE,SAAO;AACT;AAEA,SAAS,eAAe,OAA4C;AAClE,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,MAAM;AACZ,MAAI,IAAI,YAAY,OAAO,IAAI,aAAa,UAAU;AACpD,UAAM,KAAK,IAAI;AACf,QAAI,OAAO,GAAG,SAAS,SAAU,QAAO,EAAE,MAAM,GAAG,MAAM,WAAW,mBAAmB,GAAG,SAAS,EAAE;AAAA,EACvG;AACA,QAAM,OACJ,OAAO,IAAI,SAAS,WAAW,IAAI,OAC/B,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAC1C,OAAO,IAAI,cAAc,WAAW,IAAI,YACtC;AACV,MAAI,CAAC,KAAM,QAAO;AAClB,SAAO,EAAE,MAAM,WAAW,mBAAmB,IAAI,aAAa,IAAI,QAAQ,IAAI,UAAU,EAAE;AAC5F;AAEA,SAAS,mBAAmB,UAAoC;AAC9D,QAAM,SAAS,iBAAiB,QAAQ;AACxC,MAAI,MAAM,QAAQ,MAAM,EAAG,QAAO,OAAO,IAAI,cAAc,EAAE,OAAO,CAAC,SAAiC,QAAQ,IAAI,CAAC;AACnH,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,eAAW,OAAO,CAAC,kBAAkB,cAAc,OAAO,GAAG;AAC3D,UAAI,MAAM,QAAQ,IAAI,GAAG,CAAC,GAAG;AAC3B,eAAO,IAAI,GAAG,EAAE,IAAI,cAAc,EAAE,OAAO,CAAC,SAAiC,QAAQ,IAAI,CAAC;AAAA,MAC5F;AAAA,IACF;AACA,UAAM,SAAS,eAAe,GAAG;AACjC,QAAI,OAAQ,QAAO,CAAC,MAAM;AAAA,EAC5B;AACA,SAAO,CAAC;AACV;AAEA,SAAS,WAAW,gBAA2B,QAA0B;AACvE,SAAO,eAAe,KAAK,CAAC,aAAa,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM,CAAC;AAChG;AAEA,SAAS,YAAY,UAA2B,QAAiC;AAC/E,MAAI,SAAS,SAAS,OAAO,KAAM,QAAO;AAC1C,aAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,SAAS,SAAS,GAAG;AACnE,QAAI,EAAE,WAAW,OAAO,YAAY;AAClC,UAAI,QAAQ,KAAK,CAAC,UAAU,UAAU,MAAM,UAAU,QAAQ,UAAU,MAAS,EAAG;AACpF,aAAO;AAAA,IACT;AACA,QAAI,CAAC,WAAW,SAAS,OAAO,UAAU,OAAO,CAAC,EAAG,QAAO;AAAA,EAC9D;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,UAA8B;AACjE,QAAM,OAAO,SAAS,IAAI;AAC1B,QAAM,SAAS,mBAAmB,QAAQ;AAC1C,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI,UAAU;AACd,aAAW,YAAY,KAAK,UAAU;AACpC,UAAM,QAAQ,OAAO,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,YAAY,UAAU,IAAI,CAAC;AACvF,QAAI,SAAS,GAAG;AACd,WAAK,IAAI,KAAK;AACd,iBAAW;AAAA,IACb;AAAA,EACF;AACA,QAAM,SAAS,KAAK,SAAS,WAAW,IAAI,IAAI,UAAU,KAAK,SAAS;AACxE,QAAM,YAAY,OAAO,WAAW,IAAI,IAAI,UAAU,OAAO;AAC7D,QAAM,QAAQ;AACd,SAAO;AAAA,IACL,UAAU,WAAW,KAAK,cAAc;AAAA,IACxC;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,UAAU,KAAK;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B,CAAC;AAAA,EACH;AACF;AAEO,SAAS,oBAAsC;AACpD,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AAEnD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,QAAQ;AACpB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,WAAW,aAAa;AAC9B,YAAM,WAAW,SAAS,KAAK,QAAQ,GAAG,iBAAiB;AAC3D,YAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,4BAA4B;AACxE,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,QAAQ;AACpB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,UAAU;AAAA,QACpB,gBAAgB,KAAK,SAAS,IAAI,CAAC,UAAU;AAAA,UAC3C,MAAM,KAAK;AAAA,UACX,WAAW,OAAO,YAAY,OAAO,QAAQ,KAAK,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAAA,QACvG,EAAE;AAAA,MACJ,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-CWIOBFSP.js

@@ -1,197 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/toollm.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "toollm.json");
-var DEFAULT_QUERY_REL = join("data_example", "instruction", "G1_query.json");
-var toolbenchDir = () => process.env.TOOLBENCH_DIR;
-var queryFile = (dir) => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL);
-var toollmOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text.trim();
-  }
-};
-function rowToTask(row) {
-  const relevantApis = normalizeApiPairs(row["relevant APIs"] ?? []);
-  const meta = {
-    queryId: String(row.query_id),
-    apiList: row.api_list ?? [],
-    relevantApis,
-    deterministicJudge: "api-selection"
-  };
-  return {
-    id: String(row.query_id),
-    prompt: [
-      "Solve this ToolLLM/ToolBench API-use task.",
-      "Use only the listed APIs/tools and return the completed tool-use trace plus final answer.",
-      "",
-      `Query: ${row.query}`,
-      "",
-      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
-      "",
-      'Return the APIs you used as JSON: {"api_calls":[{"tool_name":"...","api_name":"..."}]}.'
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function normalizeApiPart(value) {
-  return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
-}
-function apiKey(pair) {
-  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`;
-}
-function normalizeApiPairs(value) {
-  if (!Array.isArray(value)) return [];
-  const out = [];
-  for (const item of value) {
-    if (Array.isArray(item) && typeof item[0] === "string" && typeof item[1] === "string") {
-      out.push([item[0], item[1]]);
-    } else if (item && typeof item === "object" && typeof item.tool_name === "string" && typeof item.api_name === "string") {
-      out.push([item.tool_name, item.api_name]);
-    }
-  }
-  return out;
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.relevantApis)) {
-    throw new Error(`ToolLLM task ${task.id} missing metadata \u2014 loadTasks did not populate deterministic API-selection labels`);
-  }
-  return md;
-}
-function selectRows(rows, opts) {
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-function assertDeterministicSubset(tasks, source) {
-  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id);
-  if (missing.length > 0) {
-    throw new Error(
-      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(", ")}. Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.`
-    );
-  }
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[toollm] TOOLLM_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  const tasks = selectRows(rows, opts);
-  assertDeterministicSubset(tasks, FIXTURES);
-  return tasks;
-}
-async function loadOfficialTasks(dir, opts) {
-  const source = queryFile(dir);
-  const tasks = selectRows(JSON.parse(await readFile(source, "utf8")), opts);
-  assertDeterministicSubset(tasks, source);
-  return tasks;
-}
-function extractJsonBlock(text) {
-  const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
-  const raw = (fences.at(-1)?.[1] ?? text).trim();
-  try {
-    return JSON.parse(raw);
-  } catch {
-    return void 0;
-  }
-}
-function extractCalledApis(text, expected) {
-  const parsed = extractJsonBlock(text);
-  if (parsed && typeof parsed === "object") {
-    const raw = parsed;
-    const fromApiCalls = normalizeApiPairs(raw.api_calls);
-    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: "structured-json" };
-    const fromCalls = normalizeApiPairs(raw.calls);
-    if (fromCalls.length > 0) return { pairs: fromCalls, source: "structured-json" };
-  }
-  const lower = text.toLowerCase();
-  return {
-    pairs: expected.filter(([tool, api]) => {
-      const toolNeedle = normalizeApiPart(tool);
-      const apiNeedle = normalizeApiPart(api);
-      const compactText = lower.replace(/[^a-z0-9]+/g, "");
-      return compactText.includes(`${toolNeedle}${apiNeedle}`) || lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase());
-    }),
-    source: "text-mention"
-  };
-}
-function scoreApiSelection(task, artifact) {
-  const meta = readMeta(task);
-  if (meta.relevantApis.length === 0) {
-    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`);
-  }
-  const expected = new Set(meta.relevantApis.map(apiKey));
-  const extracted = extractCalledApis(artifact, meta.relevantApis);
-  const calledPairs = extracted.pairs;
-  const called = new Set(calledPairs.map(apiKey));
-  const truePositives = [...called].filter((key) => expected.has(key)).length;
-  const precision = called.size === 0 ? 0 : truePositives / called.size;
-  const recall = truePositives / expected.size;
-  const score = expected.size === 0 ? 0 : recall;
-  const resolved = extracted.source === "structured-json" && recall === 1 && precision === 1;
-  return {
-    resolved,
-    score,
-    detail: JSON.stringify({
-      scoring: "api-selection-only",
-      extractionSource: extracted.source,
-      queryId: meta.queryId,
-      expected: meta.relevantApis,
-      called: calledPairs,
-      precision,
-      recall,
-      fullToolEvalScore: null
-    })
-  };
-}
-function createToolLlmAdapter() {
-  const fixturesMode = process.env.TOOLLM_FIXTURES === "1";
-  return {
-    name: "toollm",
-    output: toollmOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = toolbenchDir();
-      if (!dir) {
-        throw new Error("TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.");
-      }
-      await loadOfficialTasks(dir, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = toolbenchDir();
-      if (!dir) throw new Error("TOOLBENCH_DIR is required to load ToolLLM tasks");
-      return loadOfficialTasks(dir, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      if (meta.relevantApis.length === 0) return void 0;
-      return JSON.stringify({
-        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name }))
-      }, null, 2);
-    },
-    async judge(task, artifact) {
-      return scoreApiSelection(task, artifact);
-    }
-  };
-}
-
-export {
-  toollmOutput,
-  createToolLlmAdapter
-};
-//# sourceMappingURL=chunk-CWIOBFSP.js.map

package/dist/chunk-CWIOBFSP.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,IAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,IAAM,eAAe,MAA0B,QAAQ,IAAI;AAC3D,IAAM,YAAY,CAAC,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEhG,IAAM,eAAsC;AAAA,EACjD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,UAAU,KAA8B;AAC/C,QAAM,eAAe,kBAAkB,IAAI,eAAe,KAAK,CAAC,CAAC;AACjE,QAAM,OAAsB;AAAA,IAC1B,SAAS,OAAO,IAAI,QAAQ;AAAA,IAC5B,SAAS,IAAI,YAAY,CAAC;AAAA,IAC1B;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,QAAQ;AAAA,IACvB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,MACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,MAC9D;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;AAC/C,SAAO,MAAM,YAAY,EAAE,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;AACvD,SAAO,GAAG,iBAAiB,KAAK,CAAC,CAAC,CAAC,IAAI,iBAAiB,KAAK,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,kBAAkB,OAAyC;AAClE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAA+B,CAAC;AACtC,aAAW,QAAQ,OAAO;AACxB,QAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,CAAC,MAAM,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU;AACrF,UAAI,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7B,WACE,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UACxD;AACA,UAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;AAAA,IAC/F;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GAAG;AAC1C,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wFAAmF;AAAA,EAC5H;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;AACxE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;AACpF,QAAM,UAAU,MAAM,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,aAAa,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,EAAE;AACtG,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,0DAA0D,QAAQ,MAAM,IAAI,MAAM,MAAM,iBAAiB,MAAM,KAAK,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAEpJ;AAAA,EACF;AACF;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6CAAwC,KAAK,MAAM,mBAAmB;AACnF,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,4BAA0B,OAAO,QAAQ;AACzC,SAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,SAAS,UAAU,GAAG;AAC5B,QAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;AAC3F,4BAA0B,OAAO,MAAM;AACvC,SAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;AAC7F,QAAM,SAAS,iBAAiB,IAAI;AACpC,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,UAAM,eAAe,kBAAkB,IAAI,SAAS;AACpD,QAAI,aAAa,SAAS,EAAG,QAAO,EAAE,OAAO,cAAc,QAAQ,kBAAkB;AACrF,UAAM,YAAY,kBAAkB,IAAI,KAAK;AAC7C,QAAI,UAAU,SAAS,EAAG,QAAO,EAAE,OAAO,WAAW,QAAQ,kBAAkB;AAAA,EACjF;AAEA,QAAM,QAAQ,KAAK,YAAY;AAC/B,SAAO;AAAA,IACL,OAAO,SAAS,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM;AACtC,YAAM,aAAa,iBAAiB,IAAI;AACxC,YAAM,YAAY,iBAAiB,GAAG;AACtC,YAAM,cAAc,MAAM,QAAQ,eAAe,EAAE;AACnD,aAAO,YAAY,SAAS,GAAG,UAAU,GAAG,SAAS,EAAE,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;AAAA,IACrI,CAAC;AAAA,IACD,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;AACxE,QAAM,OAAO,SAAS,IAAI;AAC1B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,+DAA+D;AAAA,EACxG;AACA,QAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;AACtD,QAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;AAC/D,QAAM,cAAc,UAAU;AAC9B,QAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;AAC9C,QAAM,gBAAgB,CAAC,GAAG,MAAM,EAAE,OAAO,CAAC,QAAQ,SAAS,IAAI,GAAG,CAAC,EAAE;AACrE,QAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;AACjE,QAAM,SAAS,gBAAgB,SAAS;AACxC,QAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;AACxC,QAAM,WAAW,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;AACzF,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS;AAAA,MACT,kBAAkB,UAAU;AAAA,MAC5B,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,oBAAoB;AAErD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,sHAAsH;AAAA,MACxI;AACA,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,KAAK,aAAa,WAAW,EAAG,QAAO;AAC3C,aAAO,KAAK,UAAU;AAAA,QACpB,WAAW,KAAK,aAAa,IAAI,CAAC,CAAC,WAAW,QAAQ,OAAO,EAAE,WAAW,SAAS,EAAE;AAAA,MACvF,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,kBAAkB,MAAM,QAAQ;AAAA,IACzC;AAAA,EACF;AACF;","names":[]}