@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/benchmarks/toollm.js

@@ -1,10 +1,185 @@
-import {
-  createToolLlmAdapter,
-  toollmOutput
-} from "../chunk-CWIOBFSP.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createToolLlmAdapter,
-  toollmOutput
-};
+import { benchRoot } from "./_harness.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/toollm.ts
+/**
+* ToolLLM/ToolBench adapter.
+*
+* ToolBench task loading is useful for breadth, but the official ToolEval pass
+* rate evaluator is LLM-based and stochastic. This adapter therefore scores
+* only ToolBench's deterministic API-selection labels (`relevant APIs`). It
+* never records a full ToolEval pass-rate score.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "toollm.json");
+const DEFAULT_QUERY_REL = join("data_example", "instruction", "G1_query.json");
+const toolbenchDir = () => process.env.TOOLBENCH_DIR;
+const queryFile = (dir) => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL);
+const toollmOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return text.trim();
+} };
+function rowToTask(row) {
+	const relevantApis = normalizeApiPairs(row["relevant APIs"] ?? []);
+	const meta = {
+		queryId: String(row.query_id),
+		apiList: row.api_list ?? [],
+		relevantApis,
+		deterministicJudge: "api-selection"
+	};
+	return {
+		id: String(row.query_id),
+		prompt: [
+			"Solve this ToolLLM/ToolBench API-use task.",
+			"Use only the listed APIs/tools and return the completed tool-use trace plus final answer.",
+			"",
+			`Query: ${row.query}`,
+			"",
+			`Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
+			"",
+			"Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}."
+		].join("\n"),
+		metadata: meta
+	};
+}
+function normalizeApiPart(value) {
+	return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+function apiKey(pair) {
+	return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`;
+}
+function normalizeApiPairs(value) {
+	if (!Array.isArray(value)) return [];
+	const out = [];
+	for (const item of value) if (Array.isArray(item) && typeof item[0] === "string" && typeof item[1] === "string") out.push([item[0], item[1]]);
+	else if (item && typeof item === "object" && typeof item.tool_name === "string" && typeof item.api_name === "string") out.push([item.tool_name, item.api_name]);
+	return out;
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.relevantApis)) throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`);
+	return md;
+}
+function selectRows(rows, opts) {
+	let tasks = rows.map(rowToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`);
+	return tasks;
+}
+function assertDeterministicSubset(tasks, source) {
+	const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id);
+	if (missing.length > 0) throw new Error(`ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(", ")}. Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.`);
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	const tasks = selectRows(rows, opts);
+	assertDeterministicSubset(tasks, FIXTURES);
+	return tasks;
+}
+async function loadOfficialTasks(dir, opts) {
+	const source = queryFile(dir);
+	const tasks = selectRows(JSON.parse(await readFile(source, "utf8")), opts);
+	assertDeterministicSubset(tasks, source);
+	return tasks;
+}
+function extractJsonBlock(text) {
+	const raw = ([...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+	try {
+		return JSON.parse(raw);
+	} catch {
+		return;
+	}
+}
+function extractCalledApis(text, expected) {
+	const parsed = extractJsonBlock(text);
+	if (parsed && typeof parsed === "object") {
+		const raw = parsed;
+		const fromApiCalls = normalizeApiPairs(raw.api_calls);
+		if (fromApiCalls.length > 0) return {
+			pairs: fromApiCalls,
+			source: "structured-json"
+		};
+		const fromCalls = normalizeApiPairs(raw.calls);
+		if (fromCalls.length > 0) return {
+			pairs: fromCalls,
+			source: "structured-json"
+		};
+	}
+	const lower = text.toLowerCase();
+	return {
+		pairs: expected.filter(([tool, api]) => {
+			const toolNeedle = normalizeApiPart(tool);
+			const apiNeedle = normalizeApiPart(api);
+			return lower.replace(/[^a-z0-9]+/g, "").includes(`${toolNeedle}${apiNeedle}`) || lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase());
+		}),
+		source: "text-mention"
+	};
+}
+function scoreApiSelection(task, artifact) {
+	const meta = readMeta(task);
+	if (meta.relevantApis.length === 0) throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`);
+	const expected = new Set(meta.relevantApis.map(apiKey));
+	const extracted = extractCalledApis(artifact, meta.relevantApis);
+	const calledPairs = extracted.pairs;
+	const called = new Set(calledPairs.map(apiKey));
+	const truePositives = [...called].filter((key) => expected.has(key)).length;
+	const precision = called.size === 0 ? 0 : truePositives / called.size;
+	const recall = truePositives / expected.size;
+	const score = expected.size === 0 ? 0 : recall;
+	return {
+		resolved: extracted.source === "structured-json" && recall === 1 && precision === 1,
+		score,
+		detail: JSON.stringify({
+			scoring: "api-selection-only",
+			extractionSource: extracted.source,
+			queryId: meta.queryId,
+			expected: meta.relevantApis,
+			called: calledPairs,
+			precision,
+			recall,
+			fullToolEvalScore: null
+		})
+	};
+}
+function createToolLlmAdapter() {
+	const fixturesMode = process.env.TOOLLM_FIXTURES === "1";
+	return {
+		name: "toollm",
+		output: toollmOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = toolbenchDir();
+			if (!dir) throw new Error("TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.");
+			await loadOfficialTasks(dir, { limit: 1 });
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const dir = toolbenchDir();
+			if (!dir) throw new Error("TOOLBENCH_DIR is required to load ToolLLM tasks");
+			return loadOfficialTasks(dir, opts);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			if (meta.relevantApis.length === 0) return void 0;
+			return JSON.stringify({ api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({
+				tool_name,
+				api_name
+			})) }, null, 2);
+		},
+		async judge(task, artifact) {
+			return scoreApiSelection(task, artifact);
+		}
+	};
+}
+//#endregion
+export { createToolLlmAdapter, toollmOutput };
+
 //# sourceMappingURL=toollm.js.map

package/dist/benchmarks/toollm.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"toollm.js","names":[],"sources":["../../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,MAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,MAAM,qBAAyC,QAAQ,IAAI;AAC3D,MAAM,aAAa,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEvG,MAAa,eAAsC,EACjD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAS,UAAU,KAA8B;CAC/C,MAAM,eAAe,kBAAkB,IAAI,oBAAoB,CAAC,CAAC;CACjE,MAAM,OAAsB;EAC1B,SAAS,OAAO,IAAI,QAAQ;EAC5B,SAAS,IAAI,YAAY,CAAC;EAC1B;EACA,oBAAoB;CACtB;CACA,OAAO;EACL,IAAI,OAAO,IAAI,QAAQ;EACvB,QAAQ;GACN;GACA;GACA;GACA,UAAU,IAAI;GACd;GACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC;GAC7D;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;CAC/C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;CACvD,OAAO,GAAG,iBAAiB,KAAK,EAAE,EAAE,GAAG,iBAAiB,KAAK,EAAE;AACjE;AAEA,SAAS,kBAAkB,OAAyC;CAClE,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,QAAQ,OACjB,IAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,OAAO,YAAY,OAAO,KAAK,OAAO,UAC3E,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;MACtB,IACL,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UAExD,IAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;CAGjG,OAAO;AACT;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GACvC,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kFAAkF;CAE5H,OAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;CACxE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,GAAG;CAC3F,OAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;CACpF,MAAM,UAAU,MAAM,QAAQ,SAAS,SAAS,IAAI,CAAC,CAAC,aAAa,WAAW,CAAC,CAAC,CAAC,KAAK,SAAS,KAAK,EAAE;CACtG,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,0DAA0D,QAAQ,OAAO,GAAG,MAAM,OAAO,gBAAgB,OAAO,IAAI,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,2GAErJ;AAEJ;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,wCAAwC,KAAK,OAAO,kBAAkB;CACnF,MAAM,QAAQ,WAAW,MAAM,IAAI;CACnC,0BAA0B,OAAO,QAAQ;CACzC,OAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,SAAS,UAAU,GAAG;CAC5B,MAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;CAC3F,0BAA0B,OAAO,MAAM;CACvC,OAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,iCAAiC,CAChD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;CAC7F,MAAM,SAAS,iBAAiB,IAAI;CACpC,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,MAAM,eAAe,kBAAkB,IAAI,SAAS;EACpD,IAAI,aAAa,SAAS,GAAG,OAAO;GAAE,OAAO;GAAc,QAAQ;EAAkB;EACrF,MAAM,YAAY,kBAAkB,IAAI,KAAK;EAC7C,IAAI,UAAU,SAAS,GAAG,OAAO;GAAE,OAAO;GAAW,QAAQ;EAAkB;CACjF;CAEA,MAAM,QAAQ,KAAK,YAAY;CAC/B,OAAO;EACL,OAAO,SAAS,QAAQ,CAAC,MAAM,SAAS;GACtC,MAAM,aAAa,iBAAiB,IAAI;GACxC,MAAM,YAAY,iBAAiB,GAAG;GAEtC,OADoB,MAAM,QAAQ,eAAe,EAChC,CAAC,CAAC,SAAS,GAAG,aAAa,WAAW,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;EACrI,CAAC;EACD,QAAQ;CACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;CACxE,MAAM,OAAO,SAAS,IAAI;CAC1B,IAAI,KAAK,aAAa,WAAW,GAC/B,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,8DAA8D;CAExG,MAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;CACtD,MAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;CAC/D,MAAM,cAAc,UAAU;CAC9B,MAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;CAC9C,MAAM,gBAAgB,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,QAAQ,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC;CACrE,MAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;CACjE,MAAM,SAAS,gBAAgB,SAAS;CACxC,MAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;CAExC,OAAO;EACL,UAFe,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;EAGvF;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS;GACT,kBAAkB,UAAU;GAC5B,SAAS,KAAK;GACd,UAAU,KAAK;GACf,QAAQ;GACR;GACA;GACA,mBAAmB;EACrB,CAAC;CACH;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,sHAAsH;GAExI,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,KAAK,aAAa,WAAW,GAAG,OAAO,KAAA;GAC3C,OAAO,KAAK,UAAU,EACpB,WAAW,KAAK,aAAa,KAAK,CAAC,WAAW,eAAe;IAAE;IAAW;GAAS,EAAE,EACvF,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,kBAAkB,MAAM,QAAQ;EACzC;CACF;AACF"}

package/dist/benchmarks/trata-hedge.d.ts

@@ -1,32 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial
- * analysis tasks across 6 domains (private equity, managed care, industrials,
- * vertical SaaS, REITs, insurance). Each task bundles real earnings-call
- * transcripts, financial statements, press releases, and SEC filings under
- * `environment/data/`; the agent must produce a grounded analysis citing those
- * files.
- *
- * Harbor architecture: the original benchmark runs each task in a Docker
- * container with file-read tools. This adapter skips Harbor and embeds the data
- * files directly in the worker prompt so any router backend can score it. Large
- * files (> 30 KB) are truncated to fit within model context limits. The judge
- * reads the actual files from disk for citation verification.
- *
- * Judge: a 3-stage LLM cascade faithful to grade.py —
- *   Task 1: hallucination check (cited-file context + agent answer)
- *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)
- *   Task 3: synthesis check (ground truth + agent answer)
- * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,
- * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;
- * resolved = score 4 (sparse reward).
- *
- * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.
- * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/trata-hedge.d.ts
 declare function createTrataHedgeAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createTrataHedgeAdapter };
+//# sourceMappingURL=trata-hedge.d.ts.map

package/dist/benchmarks/trata-hedge.js

@@ -1,7 +1,337 @@
-import {
-  createTrataHedgeAdapter
-} from "../chunk-VQRS7VUC.js";
-export {
-  createTrataHedgeAdapter
-};
+import { join } from "node:path";
+import { readFileSync, readdirSync, statSync } from "node:fs";
+//#region src/benchmarks/trata-hedge.ts
+/**
+* Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial
+* analysis tasks across 6 domains (private equity, managed care, industrials,
+* vertical SaaS, REITs, insurance). Each task bundles real earnings-call
+* transcripts, financial statements, press releases, and SEC filings under
+* `environment/data/`; the agent must produce a grounded analysis citing those
+* files.
+*
+* Harbor architecture: the original benchmark runs each task in a Docker
+* container with file-read tools. This adapter skips Harbor and embeds the data
+* files directly in the worker prompt so any router backend can score it. Large
+* files (> 30 KB) are truncated to fit within model context limits. The judge
+* reads the actual files from disk for citation verification.
+*
+* Judge: a 3-stage LLM cascade faithful to grade.py —
+*   Task 1: hallucination check (cited-file context + agent answer)
+*   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)
+*   Task 3: synthesis check (ground truth + agent answer)
+* Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,
+* ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;
+* resolved = score 4 (sparse reward).
+*
+* Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.
+* Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench
+*/
+const DEFAULT_BENCH_ROOT = "/tmp/trata-hedge-bench";
+const MAX_FILE_BYTES = 8e3;
+const MAX_DATA_CHARS = 85e3;
+const WORKER_SUFFIX = [
+	"",
+	"The data files above are your only source. Cite each claim with the filename that supports it.",
+	"Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.",
+	"Take a clear position on the topic. Every factual claim must name the file it comes from."
+].join("\n");
+const JUDGE_MODEL = "gemini-2.5-pro";
+function readText(p, maxBytes = MAX_FILE_BYTES) {
+	try {
+		const buf = readFileSync(p);
+		if (buf.length <= maxBytes) return buf.toString("utf8");
+		return buf.subarray(0, maxBytes).toString("utf8") + `\n[… truncated at ${maxBytes} bytes …]`;
+	} catch {
+		return `[unreadable: ${p}]`;
+	}
+}
+function listFilesRec(dir) {
+	const out = [];
+	try {
+		for (const name of readdirSync(dir)) {
+			const full = join(dir, name);
+			try {
+				if (statSync(full).isDirectory()) out.push(...listFilesRec(full));
+				else out.push(full);
+			} catch {}
+		}
+	} catch {}
+	return out.sort();
+}
+function buildDataBlock(dataDir) {
+	const files = listFilesRec(dataDir);
+	if (files.length === 0) return "(no data files found)";
+	const priority = (f) => {
+		if (f.includes("company_profiles")) return 0;
+		if (f.includes("earnings_call")) return 1;
+		if (/financials\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2;
+		if (f.includes("sec_filings")) return 3;
+		if (f.includes("investor_pres")) return 4;
+		if (f.includes("press_release")) return 5;
+		return 6;
+	};
+	const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b));
+	const blocks = [];
+	let total = 0;
+	for (const f of sorted) {
+		const chunk = `\n=== FILE: data/${f.slice(dataDir.length + 1)} ===\n${readText(f)}`;
+		if (total + chunk.length > MAX_DATA_CHARS) {
+			blocks.push(`\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`);
+			break;
+		}
+		blocks.push(chunk);
+		total += chunk.length;
+	}
+	return blocks.join("");
+}
+function buildCitedDataBlock(answerText, dataDir) {
+	const files = listFilesRec(dataDir);
+	const cited = [];
+	for (const f of files) {
+		const rel = f.slice(dataDir.length + 1);
+		const name = rel.split("/").pop() ?? "";
+		if (answerText.includes(rel) || answerText.includes(name)) cited.push(f);
+	}
+	if (cited.length === 0) return "(The agent did not cite any data files, or no cited file could be located.)";
+	const blocks = [];
+	for (const f of cited) {
+		const rel = f.slice(dataDir.length + 1);
+		blocks.push(`\n=== FILE: data/${rel} ===\n${readText(f)}`);
+	}
+	return blocks.join("");
+}
+function parseThemeMoveCounts(groundTruth) {
+	const lines = groundTruth.split("\n");
+	const themeStarts = lines.reduce((acc, l, i) => {
+		if (/^\d+\./.test(l.trim())) acc.push(i);
+		return acc;
+	}, []);
+	return themeStarts.map((start, k) => {
+		const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length;
+		const block = lines.slice(start, end).join("\n");
+		return {
+			label: lines[start].replace(/^\d+\.\s*/, "").trim(),
+			nMoves: (block.match(/^\s*\[[a-z]\]\s+/gm) ?? []).length
+		};
+	});
+}
+function coverageThreshold(nMoves) {
+	return Math.max(1, Math.min(nMoves - 1, 3));
+}
+function judgeRouter() {
+	const key = process.env.TANGLE_API_KEY;
+	if (!key) throw new Error("TANGLE_API_KEY is required for the Trata hedge-bench judge");
+	const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL;
+	return {
+		baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+		key,
+		model
+	};
+}
+function parseJsonFallback(raw) {
+	let s = raw.trim();
+	if (s.startsWith("```")) s = s.split("\n").slice(1).join("\n");
+	if (s.endsWith("```")) s = s.slice(0, s.lastIndexOf("```"));
+	s = s.replace(/,(\s*[}\]])/g, "$1").trim();
+	try {
+		return JSON.parse(s);
+	} catch {
+		try {
+			const m = s.match(/\{[\s\S]*\}/);
+			if (m) return JSON.parse(m[0]);
+		} catch {}
+	}
+	return null;
+}
+async function callJudge(router, prompt, maxAttempts = 2) {
+	for (let i = 0; i < maxAttempts; i++) {
+		const res = await fetch(`${router.baseUrl}/chat/completions`, {
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${router.key}`
+			},
+			body: JSON.stringify({
+				model: router.model,
+				temperature: 0,
+				max_tokens: 16384,
+				messages: [{
+					role: "user",
+					content: prompt
+				}]
+			})
+		});
+		if (!res.ok) {
+			if (i < maxAttempts - 1) continue;
+			throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+		}
+		const content = (await res.json()).choices?.[0]?.message?.content;
+		if (typeof content !== "string") continue;
+		const parsed = parseJsonFallback(content);
+		if (parsed !== null) return parsed;
+	}
+	return null;
+}
+async function task1HallucinationCheck(meta, answer, router) {
+	const dataContext = `The block below contains the contents of the data files the agent cited in their answer. Treat this as the ground truth for factual verification.
+
+<cited_data>\n${buildCitedDataBlock(answer, meta.dataDir)}\n</cited_data>`;
+	return await callJudge(router, meta.gradingTask1.replace("{data_context}", dataContext).replace("{agent_answer}", answer)) ?? {
+		hallucinations_detected: false,
+		unverifiable_claims: []
+	};
+}
+async function task2PerMoveCheck(meta, answer, flaggedClaims, router) {
+	const claimsBlock = flaggedClaims && flaggedClaims.length > 0 ? JSON.stringify(flaggedClaims, null, 2) : "(No claims were flagged as hallucinations.)";
+	return await callJudge(router, meta.gradingTask2.replace("{ground_truth}", meta.groundTruth).replace("{agent_answer}", answer).replace("{flagged_claims_block}", claimsBlock).replace("{num_themes}", String(meta.themeMoveCounts.length))) ?? { themes: [] };
+}
+async function task3SynthesisCheck(meta, answer, router) {
+	return await callJudge(router, meta.gradingTask3.replace("{ground_truth}", meta.groundTruth).replace("{agent_answer}", answer)) ?? { synthesis_found: null };
+}
+async function runTriataJudge(meta, answer, router) {
+	if (!answer.trim()) return {
+		resolved: false,
+		score: 0,
+		detail: JSON.stringify({ error: "empty answer" })
+	};
+	const t1 = await task1HallucinationCheck(meta, answer, router);
+	const flaggedClaims = t1.unverifiable_claims ?? [];
+	const [t2, t3] = await Promise.all([task2PerMoveCheck(meta, answer, flaggedClaims, router), task3SynthesisCheck(meta, answer, router)]);
+	const judgeThemes = t2.themes ?? [];
+	const themesHit = [];
+	const themesMissed = [];
+	for (let i = 0; i < meta.themeMoveCounts.length; i++) {
+		const { label, nMoves } = meta.themeMoveCounts[i];
+		const judge = judgeThemes[i] ?? {};
+		const movesHit = (judge.moves_hit ?? []).map((m) => m.replace(/[\[\]]/g, ""));
+		const tainted = new Set((judge.moves_tainted ?? []).map((m) => m.replace(/[\[\]]/g, "")));
+		const validHit = movesHit.filter((m) => !tainted.has(m));
+		const threshold = coverageThreshold(nMoves);
+		if (validHit.length >= threshold) themesHit.push(label);
+		else themesMissed.push(label);
+	}
+	const numThemes = meta.themeMoveCounts.length;
+	const synth = t3.synthesis_found;
+	const hasSynthesis = typeof synth === "string" && synth.length > 0 && synth !== "null";
+	let rawScore;
+	if (numThemes === 0) rawScore = 0;
+	else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4;
+	else if (themesHit.length === numThemes) rawScore = 3;
+	else if (themesHit.length >= 2) rawScore = 2;
+	else if (themesHit.length >= 1) rawScore = 1;
+	else rawScore = 0;
+	const hallRaw = t1.hallucinations_detected;
+	const hallucinations = hallRaw === true || hallRaw === "true";
+	return {
+		resolved: rawScore === 4,
+		score: numThemes > 0 ? rawScore / 4 : 0,
+		detail: JSON.stringify({
+			rawScore,
+			themesHit,
+			themesMissed,
+			hallucinations,
+			unverifiableClaims: flaggedClaims.length,
+			synthesis: synth ?? null,
+			judgeModel: router.model
+		})
+	};
+}
+function benchRoot() {
+	return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT;
+}
+function loadTask(taskId) {
+	const taskDir = join(benchRoot(), "environments", taskId);
+	const dataDir = join(taskDir, "environment", "data");
+	const testsDir = join(taskDir, "tests");
+	const instruction = readText(join(taskDir, "instruction.md"), Infinity);
+	const dataBlock = buildDataBlock(dataDir);
+	const groundTruth = readText(join(testsDir, "ground_truth.txt"), Infinity);
+	const gradingTask1 = readText(join(testsDir, "grading_prompt_task1.md"), Infinity);
+	const gradingTask2 = readText(join(testsDir, "grading_prompt_task2.md"), Infinity);
+	const gradingTask3 = readText(join(testsDir, "grading_prompt_task3.md"), Infinity);
+	const themeMoveCounts = parseThemeMoveCounts(groundTruth);
+	const prompt = [
+		instruction.replace("You are a financial analyst with access to the data in `/app/data/`.", "You are a financial analyst. The relevant financial data files are provided inline below.").replace(/Write your full analysis to `\/app\/answer\.txt`\.?/, "Write your full analysis below.").replace(/`\/app\/data\/`/g, "the data files below"),
+		"",
+		"## Data files (inline)",
+		dataBlock,
+		WORKER_SUFFIX
+	].join("\n");
+	const meta = {
+		taskDir,
+		taskId,
+		dataDir,
+		groundTruth,
+		gradingTask1,
+		gradingTask2,
+		gradingTask3,
+		themeMoveCounts
+	};
+	return {
+		task: {
+			id: taskId,
+			prompt,
+			metadata: meta
+		},
+		meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.taskDir !== "string" || typeof md.groundTruth !== "string") throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`);
+	return md;
+}
+/** List all task ids from the environments directory. */
+function listTaskIds(root) {
+	try {
+		return readdirSync(join(root, "environments")).filter((name) => {
+			try {
+				return statSync(join(root, "environments", name)).isDirectory();
+			} catch {
+				return false;
+			}
+		}).sort();
+	} catch (err) {
+		throw new Error(`Trata benchmark environments not found at ${join(root, "environments")}: ${err instanceof Error ? err.message : err}.\nClone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`);
+	}
+}
+function extractAnalysis(artifact) {
+	if (artifact.startsWith("ANALYSIS:")) return artifact.slice(9).trim();
+	const marker = artifact.indexOf("\nANALYSIS:");
+	if (marker !== -1) return artifact.slice(marker + 10).trim();
+	return "";
+}
+function createTrataHedgeAdapter() {
+	return {
+		name: "trata-hedge",
+		async preflight() {
+			const root = benchRoot();
+			const envDir = join(root, "environments");
+			try {
+				if (!statSync(envDir).isDirectory()) throw new Error("not a directory");
+			} catch {
+				throw new Error(`Trata hedge-bench not found at ${envDir}.\nClone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`);
+			}
+			judgeRouter();
+			const ids = listTaskIds(root);
+			if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`);
+			console.warn(`[trata-hedge] ${ids.length} tasks loaded from ${root}. Worker model reads embedded data inline (Harbor/Docker not required). Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`);
+		},
+		async loadTasks(opts = {}) {
+			let ids = listTaskIds(benchRoot());
+			if (opts.ids) {
+				const want = new Set(opts.ids);
+				ids = ids.filter((id) => want.has(id));
+			} else if (opts.limit !== void 0) ids = ids.slice(0, opts.limit);
+			return ids.map((id) => loadTask(id).task);
+		},
+		async goldArtifact(_task) {},
+		async judge(task, artifact) {
+			return runTriataJudge(readMeta(task), extractAnalysis(artifact), judgeRouter());
+		}
+	};
+}
+//#endregion
+export { createTrataHedgeAdapter };
+
 //# sourceMappingURL=trata-hedge.js.map