@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/bfcl.js

@@ -1,10 +1,265 @@
-import {
-  bfclOutput,
-  createBfclAdapter
-} from "../chunk-KP5KD6EN.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  bfclOutput,
-  createBfclAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/bfcl.ts
+/**
+* Berkeley Function Calling Leaderboard adapter.
+*
+* Scope: deterministic function-call ground-truth categories from the official
+* BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic
+* web-search/memory categories and BFCL's own model-response harness remain
+* upstream responsibilities. The adapter loads official JSONL rows plus their
+* `possible_answer` file and scores structured function-call artifacts against
+* allowed function/argument values.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "bfcl.json");
+const DEFAULT_CATEGORY = "BFCL_v4_simple_python";
+const bfclDir = () => process.env.BFCL_DIR;
+const bfclCategory = () => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY;
+const bfclOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return text.trim();
+} };
+async function assertPath(path, label) {
+	try {
+		await stat(path);
+	} catch (err) {
+		throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+function readJsonl(raw) {
+	return raw.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+}
+function taskFile(dir, category) {
+	return process.env.BFCL_DATA_FILE ?? join(dir, "bfcl_eval", "data", `${category}.json`);
+}
+function answerFile(dir, category) {
+	return process.env.BFCL_ANSWER_FILE ?? join(dir, "bfcl_eval", "data", "possible_answer", `${category}.json`);
+}
+function questionText(question) {
+	if (typeof question === "string") return question;
+	if (!Array.isArray(question)) return JSON.stringify(question, null, 2);
+	const turns = [];
+	for (const item of question.flat(3)) if (item && typeof item === "object") {
+		const role = typeof item.role === "string" ? item.role : "user";
+		const content = item.content;
+		if (typeof content === "string") turns.push(`${role}: ${content}`);
+	}
+	return turns.length > 0 ? turns.join("\n") : JSON.stringify(question, null, 2);
+}
+function normalizeScalar(value) {
+	if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)));
+	if (typeof value === "string") return value.trim().toLowerCase();
+	return JSON.stringify(value);
+}
+function normalizeAllowed(value) {
+	return Array.isArray(value) ? value : [value];
+}
+function groundTruthToCalls(value) {
+	if (!Array.isArray(value)) return [];
+	const out = [];
+	for (const item of value) {
+		if (!item || typeof item !== "object") continue;
+		for (const [name, args] of Object.entries(item)) {
+			if (!args || typeof args !== "object" || Array.isArray(args)) continue;
+			const normalized = {};
+			for (const [argName, allowed] of Object.entries(args)) normalized[argName] = normalizeAllowed(allowed);
+			out.push({
+				name,
+				arguments: normalized
+			});
+		}
+	}
+	return out;
+}
+function rowToTask(row, answer, category) {
+	const expected = groundTruthToCalls(answer.ground_truth);
+	if (expected.length === 0) throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`);
+	const meta = {
+		rowId: row.id,
+		category,
+		functions: row.function,
+		expected,
+		scoring: "bfcl-ground-truth-subset"
+	};
+	return {
+		id: row.id,
+		split: category,
+		prompt: [
+			"Solve this BFCL function-calling task.",
+			"Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.",
+			"",
+			questionText(row.question),
+			"",
+			`Available functions: ${JSON.stringify(row.function, null, 2)}`
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.expected)) throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`);
+	return md;
+}
+function selectTasks(tasks, opts) {
+	let out = tasks;
+	if (opts.split) out = out.filter((task) => task.split === opts.split);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		out = out.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) out = out.slice(0, opts.limit);
+	if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`);
+	return out;
+}
+function buildTasks(rows, answers, category, opts) {
+	const byId = new Map(answers.map((answer) => [answer.id, answer]));
+	return selectTasks(rows.map((row) => {
+		const answer = byId.get(row.id);
+		if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`);
+		return rowToTask(row, answer, category);
+	}), opts);
+}
+async function loadFixtures(opts) {
+	const fixture = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`);
+	return buildTasks(fixture.rows, fixture.answers, fixture.category, opts);
+}
+async function loadOfficialTasks(dir, opts) {
+	const category = bfclCategory();
+	const dataPath = taskFile(dir, category);
+	const answersPath = answerFile(dir, category);
+	const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, "utf8"), readFile(answersPath, "utf8")]);
+	return buildTasks(readJsonl(rawRows), readJsonl(rawAnswers), category, opts);
+}
+function extractJsonBlock(text) {
+	const raw = ([...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+	try {
+		return JSON.parse(raw);
+	} catch {
+		return;
+	}
+}
+function normalizeArguments(value) {
+	if (typeof value === "string") try {
+		return normalizeArguments(JSON.parse(value));
+	} catch {
+		return {};
+	}
+	if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+	return value;
+}
+function callFromObject(value) {
+	if (!value || typeof value !== "object") return void 0;
+	const raw = value;
+	if (raw.function && typeof raw.function === "object") {
+		const fn = raw.function;
+		if (typeof fn.name === "string") return {
+			name: fn.name,
+			arguments: normalizeArguments(fn.arguments)
+		};
+	}
+	const name = typeof raw.name === "string" ? raw.name : typeof raw.function_name === "string" ? raw.function_name : typeof raw.tool_name === "string" ? raw.tool_name : void 0;
+	if (!name) return void 0;
+	return {
+		name,
+		arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters)
+	};
+}
+function extractActualCalls(artifact) {
+	const parsed = extractJsonBlock(artifact);
+	if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call) => Boolean(call));
+	if (parsed && typeof parsed === "object") {
+		const raw = parsed;
+		for (const key of [
+			"function_calls",
+			"tool_calls",
+			"calls"
+		]) if (Array.isArray(raw[key])) return raw[key].map(callFromObject).filter((call) => Boolean(call));
+		const single = callFromObject(raw);
+		if (single) return [single];
+	}
+	return [];
+}
+function argMatches(expectedValues, actual) {
+	return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual));
+}
+function callMatches(expected, actual) {
+	if (expected.name !== actual.name) return false;
+	for (const [argName, allowed] of Object.entries(expected.arguments)) {
+		if (!(argName in actual.arguments)) {
+			if (allowed.some((value) => value === "" || value === null || value === void 0)) continue;
+			return false;
+		}
+		if (!argMatches(allowed, actual.arguments[argName])) return false;
+	}
+	return true;
+}
+function scoreCalls(task, artifact) {
+	const meta = readMeta(task);
+	const actual = extractActualCalls(artifact);
+	const used = /* @__PURE__ */ new Set();
+	let matched = 0;
+	for (const expected of meta.expected) {
+		const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call));
+		if (index >= 0) {
+			used.add(index);
+			matched += 1;
+		}
+	}
+	const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length;
+	const precision = actual.length === 0 ? 0 : matched / actual.length;
+	return {
+		resolved: recall === 1 && precision === 1,
+		score: recall,
+		detail: JSON.stringify({
+			scoring: meta.scoring,
+			category: meta.category,
+			expected: meta.expected,
+			actual,
+			precision,
+			recall,
+			fullBfclLeaderboardScore: null
+		})
+	};
+}
+function createBfclAdapter() {
+	const fixturesMode = process.env.BFCL_FIXTURES === "1";
+	return {
+		name: "bfcl",
+		output: bfclOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = bfclDir();
+			if (!dir) throw new Error("BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.");
+			const category = bfclCategory();
+			await assertPath(taskFile(dir, category), "BFCL task JSONL");
+			await assertPath(answerFile(dir, category), "BFCL possible_answer JSONL");
+			await loadOfficialTasks(dir, { limit: 1 });
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const dir = bfclDir();
+			if (!dir) throw new Error("BFCL_DIR is required to load official BFCL rows");
+			return loadOfficialTasks(dir, opts);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			return JSON.stringify({ function_calls: meta.expected.map((call) => ({
+				name: call.name,
+				arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]]))
+			})) }, null, 2);
+		},
+		async judge(task, artifact) {
+			return scoreCalls(task, artifact);
+		}
+	};
+}
+//#endregion
+export { bfclOutput, createBfclAdapter };
+
 //# sourceMappingURL=bfcl.js.map

package/dist/benchmarks/bfcl.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"bfcl.js","names":[],"sources":["../../src/benchmarks/bfcl.ts"],"sourcesContent":["/**\n * Berkeley Function Calling Leaderboard adapter.\n *\n * Scope: deterministic function-call ground-truth categories from the official\n * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic\n * web-search/memory categories and BFCL's own model-response harness remain\n * upstream responsibilities. The adapter loads official JSONL rows plus their\n * `possible_answer` file and scores structured function-call artifacts against\n * allowed function/argument values.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'bfcl.json')\nconst DEFAULT_CATEGORY = 'BFCL_v4_simple_python'\n\ninterface BfclRow {\n  id: string\n  question: unknown\n  function: unknown\n}\n\ninterface BfclAnswerRow {\n  id: string\n  ground_truth: unknown\n}\n\ninterface BfclAllowedCall {\n  name: string\n  arguments: Record<string, unknown[]>\n}\n\ninterface BfclMeta {\n  rowId: string\n  category: string\n  functions: unknown\n  expected: BfclAllowedCall[]\n  scoring: 'bfcl-ground-truth-subset'\n}\n\ninterface BfclActualCall {\n  name: string\n  arguments: Record<string, unknown>\n}\n\nconst bfclDir = (): string | undefined => process.env.BFCL_DIR\nconst bfclCategory = (): string => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY\n\nexport const bfclOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nasync function assertPath(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readJsonl(raw: string): unknown[] {\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nfunction taskFile(dir: string, category: string): string {\n  return process.env.BFCL_DATA_FILE ?? join(dir, 'bfcl_eval', 'data', `${category}.json`)\n}\n\nfunction answerFile(dir: string, category: string): string {\n  return process.env.BFCL_ANSWER_FILE ?? join(dir, 'bfcl_eval', 'data', 'possible_answer', `${category}.json`)\n}\n\nfunction questionText(question: unknown): string {\n  if (typeof question === 'string') return question\n  if (!Array.isArray(question)) return JSON.stringify(question, null, 2)\n  const turns: string[] = []\n  for (const item of question.flat(3)) {\n    if (item && typeof item === 'object') {\n      const role = typeof (item as { role?: unknown }).role === 'string' ? (item as { role: string }).role : 'user'\n      const content = (item as { content?: unknown }).content\n      if (typeof content === 'string') turns.push(`${role}: ${content}`)\n    }\n  }\n  return turns.length > 0 ? turns.join('\\n') : JSON.stringify(question, null, 2)\n}\n\nfunction normalizeScalar(value: unknown): string {\n  if (typeof value === 'number') return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)))\n  if (typeof value === 'string') return value.trim().toLowerCase()\n  return JSON.stringify(value)\n}\n\nfunction normalizeAllowed(value: unknown): unknown[] {\n  return Array.isArray(value) ? value : [value]\n}\n\nfunction groundTruthToCalls(value: unknown): BfclAllowedCall[] {\n  if (!Array.isArray(value)) return []\n  const out: BfclAllowedCall[] = []\n  for (const item of value) {\n    if (!item || typeof item !== 'object') continue\n    for (const [name, args] of Object.entries(item as Record<string, unknown>)) {\n      if (!args || typeof args !== 'object' || Array.isArray(args)) continue\n      const normalized: Record<string, unknown[]> = {}\n      for (const [argName, allowed] of Object.entries(args as Record<string, unknown>)) {\n        normalized[argName] = normalizeAllowed(allowed)\n      }\n      out.push({ name, arguments: normalized })\n    }\n  }\n  return out\n}\n\nfunction rowToTask(row: BfclRow, answer: BfclAnswerRow, category: string): BenchTask {\n  const expected = groundTruthToCalls(answer.ground_truth)\n  if (expected.length === 0) {\n    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`)\n  }\n  const meta: BfclMeta = {\n    rowId: row.id,\n    category,\n    functions: row.function,\n    expected,\n    scoring: 'bfcl-ground-truth-subset',\n  }\n  return {\n    id: row.id,\n    split: category,\n    prompt: [\n      'Solve this BFCL function-calling task.',\n      'Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.',\n      '',\n      questionText(row.question),\n      '',\n      `Available functions: ${JSON.stringify(row.function, null, 2)}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): BfclMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.expected)) {\n    throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`)\n  }\n  return md as unknown as BfclMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  let out = tasks\n  if (opts.split) out = out.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    out = out.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    out = out.slice(0, opts.limit)\n  }\n  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`)\n  return out\n}\n\nfunction buildTasks(rows: BfclRow[], answers: BfclAnswerRow[], category: string, opts: LoadOptions): BenchTask[] {\n  const byId = new Map(answers.map((answer) => [answer.id, answer]))\n  const tasks = rows.map((row) => {\n    const answer = byId.get(row.id)\n    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`)\n    return rowToTask(row, answer, category)\n  })\n  return selectTasks(tasks, opts)\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixture = JSON.parse(await readFile(FIXTURES, 'utf8')) as { rows: BfclRow[]; answers: BfclAnswerRow[]; category: string }\n  console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`)\n  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const category = bfclCategory()\n  const dataPath = taskFile(dir, category)\n  const answersPath = answerFile(dir, category)\n  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, 'utf8'), readFile(answersPath, 'utf8')])\n  const rows = readJsonl(rawRows) as BfclRow[]\n  const answers = readJsonl(rawAnswers) as BfclAnswerRow[]\n  return buildTasks(rows, answers, category, opts)\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\nfunction normalizeArguments(value: unknown): Record<string, unknown> {\n  if (typeof value === 'string') {\n    try {\n      return normalizeArguments(JSON.parse(value))\n    } catch {\n      return {}\n    }\n  }\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return {}\n  return value as Record<string, unknown>\n}\n\nfunction callFromObject(value: unknown): BfclActualCall | undefined {\n  if (!value || typeof value !== 'object') return undefined\n  const raw = value as Record<string, unknown>\n  if (raw.function && typeof raw.function === 'object') {\n    const fn = raw.function as Record<string, unknown>\n    if (typeof fn.name === 'string') return { name: fn.name, arguments: normalizeArguments(fn.arguments) }\n  }\n  const name =\n    typeof raw.name === 'string' ? raw.name\n      : typeof raw.function_name === 'string' ? raw.function_name\n        : typeof raw.tool_name === 'string' ? raw.tool_name\n          : undefined\n  if (!name) return undefined\n  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) }\n}\n\nfunction extractActualCalls(artifact: string): BfclActualCall[] {\n  const parsed = extractJsonBlock(artifact)\n  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    for (const key of ['function_calls', 'tool_calls', 'calls']) {\n      if (Array.isArray(raw[key])) {\n        return raw[key].map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n      }\n    }\n    const single = callFromObject(raw)\n    if (single) return [single]\n  }\n  return []\n}\n\nfunction argMatches(expectedValues: unknown[], actual: unknown): boolean {\n  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual))\n}\n\nfunction callMatches(expected: BfclAllowedCall, actual: BfclActualCall): boolean {\n  if (expected.name !== actual.name) return false\n  for (const [argName, allowed] of Object.entries(expected.arguments)) {\n    if (!(argName in actual.arguments)) {\n      if (allowed.some((value) => value === '' || value === null || value === undefined)) continue\n      return false\n    }\n    if (!argMatches(allowed, actual.arguments[argName])) return false\n  }\n  return true\n}\n\nfunction scoreCalls(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  const actual = extractActualCalls(artifact)\n  const used = new Set<number>()\n  let matched = 0\n  for (const expected of meta.expected) {\n    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call))\n    if (index >= 0) {\n      used.add(index)\n      matched += 1\n    }\n  }\n  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length\n  const precision = actual.length === 0 ? 0 : matched / actual.length\n  const score = recall\n  return {\n    resolved: recall === 1 && precision === 1,\n    score,\n    detail: JSON.stringify({\n      scoring: meta.scoring,\n      category: meta.category,\n      expected: meta.expected,\n      actual,\n      precision,\n      recall,\n      fullBfclLeaderboardScore: null,\n    }),\n  }\n}\n\nexport function createBfclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.BFCL_FIXTURES === '1'\n\n  return {\n    name: 'bfcl',\n    output: bfclOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = bfclDir()\n      if (!dir) {\n        throw new Error(\n          'BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.',\n        )\n      }\n      const category = bfclCategory()\n      await assertPath(taskFile(dir, category), 'BFCL task JSONL')\n      await assertPath(answerFile(dir, category), 'BFCL possible_answer JSONL')\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = bfclDir()\n      if (!dir) throw new Error('BFCL_DIR is required to load official BFCL rows')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return JSON.stringify({\n        function_calls: meta.expected.map((call) => ({\n          name: call.name,\n          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]])),\n        })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreCalls(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AACxD,MAAM,mBAAmB;AA+BzB,MAAM,gBAAoC,QAAQ,IAAI;AACtD,MAAM,qBAA6B,QAAQ,IAAI,iBAAiB;AAEhE,MAAa,aAAoC,EAC/C,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iBAAiB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACnG;AACF;AAEA,SAAS,UAAU,KAAwB;CACzC,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAS,SAAS,KAAa,UAA0B;CACvD,OAAO,QAAQ,IAAI,kBAAkB,KAAK,KAAK,aAAa,QAAQ,GAAG,SAAS,MAAM;AACxF;AAEA,SAAS,WAAW,KAAa,UAA0B;CACzD,OAAO,QAAQ,IAAI,oBAAoB,KAAK,KAAK,aAAa,QAAQ,mBAAmB,GAAG,SAAS,MAAM;AAC7G;AAEA,SAAS,aAAa,UAA2B;CAC/C,IAAI,OAAO,aAAa,UAAU,OAAO;CACzC,IAAI,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC;CACrE,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,QAAQ,SAAS,KAAK,CAAC,GAChC,IAAI,QAAQ,OAAO,SAAS,UAAU;EACpC,MAAM,OAAO,OAAQ,KAA4B,SAAS,WAAY,KAA0B,OAAO;EACvG,MAAM,UAAW,KAA+B;EAChD,IAAI,OAAO,YAAY,UAAU,MAAM,KAAK,GAAG,KAAK,IAAI,SAAS;CACnE;CAEF,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,UAAU,UAAU,MAAM,CAAC;AAC/E;AAEA,SAAS,gBAAgB,OAAwB;CAC/C,IAAI,OAAO,UAAU,UAAU,OAAO,OAAO,UAAU,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;CAC/G,IAAI,OAAO,UAAU,UAAU,OAAO,MAAM,KAAK,CAAC,CAAC,YAAY;CAC/D,OAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,iBAAiB,OAA2B;CACnD,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAC9C;AAEA,SAAS,mBAAmB,OAAmC;CAC7D,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;EACvC,KAAK,MAAM,CAAC,MAAM,SAAS,OAAO,QAAQ,IAA+B,GAAG;GAC1E,IAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;GAC9D,MAAM,aAAwC,CAAC;GAC/C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,IAA+B,GAC7E,WAAW,WAAW,iBAAiB,OAAO;GAEhD,IAAI,KAAK;IAAE;IAAM,WAAW;GAAW,CAAC;EAC1C;CACF;CACA,OAAO;AACT;AAEA,SAAS,UAAU,KAAc,QAAuB,UAA6B;CACnF,MAAM,WAAW,mBAAmB,OAAO,YAAY;CACvD,IAAI,SAAS,WAAW,GACtB,MAAM,IAAI,MAAM,QAAQ,IAAI,GAAG,0DAA0D;CAE3F,MAAM,OAAiB;EACrB,OAAO,IAAI;EACX;EACA,WAAW,IAAI;EACf;EACA,SAAS;CACX;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI,QAAQ;GACzB;GACA,wBAAwB,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC;EAC9D,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,QAAQ,GACnC,MAAM,IAAI,MAAM,aAAa,KAAK,GAAG,8DAA8D;CAErG,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,MAAM;CACV,IAAI,KAAK,OAAO,MAAM,IAAI,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACpE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,MAAM,IAAI,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAC9C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;CAE/B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,0BAA0B,KAAK,UAAU,IAAI,GAAG;CACtF,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,SAA0B,UAAkB,MAAgC;CAC/G,MAAM,OAAO,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;CAMjE,OAAO,YALO,KAAK,KAAK,QAAQ;EAC9B,MAAM,SAAS,KAAK,IAAI,IAAI,EAAE;EAC9B,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,QAAQ,SAAS,gCAAgC,IAAI,IAAI;EACtF,OAAO,UAAU,KAAK,QAAQ,QAAQ;CACxC,CACuB,GAAG,IAAI;AAChC;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC3D,QAAQ,KAAK,oCAAoC,QAAQ,KAAK,OAAO,kBAAkB;CACvF,OAAO,WAAW,QAAQ,MAAM,QAAQ,SAAS,QAAQ,UAAU,IAAI;AACzE;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,WAAW,aAAa;CAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ;CACvC,MAAM,cAAc,WAAW,KAAK,QAAQ;CAC5C,MAAM,CAAC,SAAS,cAAc,MAAM,QAAQ,IAAI,CAAC,SAAS,UAAU,MAAM,GAAG,SAAS,aAAa,MAAM,CAAC,CAAC;CAG3G,OAAO,WAFM,UAAU,OAEF,GADL,UAAU,UACI,GAAG,UAAU,IAAI;AACjD;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,+BAA+B,CAC9C,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;CACnE,IAAI,OAAO,UAAU,UACnB,IAAI;EACF,OAAO,mBAAmB,KAAK,MAAM,KAAK,CAAC;CAC7C,QAAQ;EACN,OAAO,CAAC;CACV;CAEF,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACzE,OAAO;AACT;AAEA,SAAS,eAAe,OAA4C;CAClE,IAAI,CAAC,SAAS,OAAO,UAAU,UAAU,OAAO,KAAA;CAChD,MAAM,MAAM;CACZ,IAAI,IAAI,YAAY,OAAO,IAAI,aAAa,UAAU;EACpD,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,GAAG,SAAS,UAAU,OAAO;GAAE,MAAM,GAAG;GAAM,WAAW,mBAAmB,GAAG,SAAS;EAAE;CACvG;CACA,MAAM,OACJ,OAAO,IAAI,SAAS,WAAW,IAAI,OAC/B,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAC1C,OAAO,IAAI,cAAc,WAAW,IAAI,YACtC,KAAA;CACV,IAAI,CAAC,MAAM,OAAO,KAAA;CAClB,OAAO;EAAE;EAAM,WAAW,mBAAmB,IAAI,aAAa,IAAI,QAAQ,IAAI,UAAU;CAAE;AAC5F;AAEA,SAAS,mBAAmB,UAAoC;CAC9D,MAAM,SAAS,iBAAiB,QAAQ;CACxC,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO,OAAO,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;CACnH,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,KAAK,MAAM,OAAO;GAAC;GAAkB;GAAc;EAAO,GACxD,IAAI,MAAM,QAAQ,IAAI,IAAI,GACxB,OAAO,IAAI,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;EAG9F,MAAM,SAAS,eAAe,GAAG;EACjC,IAAI,QAAQ,OAAO,CAAC,MAAM;CAC5B;CACA,OAAO,CAAC;AACV;AAEA,SAAS,WAAW,gBAA2B,QAA0B;CACvE,OAAO,eAAe,MAAM,aAAa,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM,CAAC;AAChG;AAEA,SAAS,YAAY,UAA2B,QAAiC;CAC/E,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAC1C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,SAAS,SAAS,GAAG;EACnE,IAAI,EAAE,WAAW,OAAO,YAAY;GAClC,IAAI,QAAQ,MAAM,UAAU,UAAU,MAAM,UAAU,QAAQ,UAAU,KAAA,CAAS,GAAG;GACpF,OAAO;EACT;EACA,IAAI,CAAC,WAAW,SAAS,OAAO,UAAU,QAAQ,GAAG,OAAO;CAC9D;CACA,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,UAA8B;CACjE,MAAM,OAAO,SAAS,IAAI;CAC1B,MAAM,SAAS,mBAAmB,QAAQ;CAC1C,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,UAAU;CACd,KAAK,MAAM,YAAY,KAAK,UAAU;EACpC,MAAM,QAAQ,OAAO,WAAW,MAAM,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,YAAY,UAAU,IAAI,CAAC;EACvF,IAAI,SAAS,GAAG;GACd,KAAK,IAAI,KAAK;GACd,WAAW;EACb;CACF;CACA,MAAM,SAAS,KAAK,SAAS,WAAW,IAAI,IAAI,UAAU,KAAK,SAAS;CACxE,MAAM,YAAY,OAAO,WAAW,IAAI,IAAI,UAAU,OAAO;CAE7D,OAAO;EACL,UAAU,WAAW,KAAK,cAAc;EACxC,OAAA;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS,KAAK;GACd,UAAU,KAAK;GACf,UAAU,KAAK;GACf;GACA;GACA;GACA,0BAA0B;EAC5B,CAAC;CACH;AACF;AAEA,SAAgB,oBAAsC;CACpD,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CAEnD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qMACF;GAEF,MAAM,WAAW,aAAa;GAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ,GAAG,iBAAiB;GAC3D,MAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,4BAA4B;GACxE,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,UAAU,EACpB,gBAAgB,KAAK,SAAS,KAAK,UAAU;IAC3C,MAAM,KAAK;IACX,WAAW,OAAO,YAAY,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,YAAY,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;GACvG,EAAE,EACJ,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,WAAW,MAAM,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/cad-design.d.ts

@@ -1,45 +1,20 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the
- * REAL OpenSCAD engine: compile + export STL, then measure the produced solid
- * geometry against the task's spec. Fully deterministic — the authoritative CAD
- * kernel is the gate, not an LLM and not the agent's own claim.
- *
- * This is verifiable-reward CAD: an agent is far better at *writing* exact
- * parametric code than at clicking a GUI, and the kernel either produces the
- * specified geometry or it doesn't. Every judged artifact also renders a PNG,
- * so a refine loop's attempts become a watchable "model getting better" reel
- * (run-capsule consumes the renders).
- *
- * Spec checks are geometric and ungameable:
- *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)
- *   - volumes       : disconnected-solid count within [min,max]
- *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)
- *   - detail        : triangle count ≥ floor (not a degenerate single cube)
- *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat
- *                     box can't fake this; a real gabled/hipped roof tapers
- *   - hollow        : interior cavity present (walls, not a solid block) — the
- *                     bbox volume materially exceeds the printed solid volume
- *
- * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/cad-design.d.ts
 /** Spec assertions a CAD task can require. All are deterministic + geometric. */
 interface CadSpec {
-    volumes?: [number, number];
-    bbox?: {
-        x?: [number, number];
-        y?: [number, number];
-        z?: [number, number];
-    };
-    minTriangles?: number;
-    /** Top-band XY footprint must be < this fraction of the base footprint. */
-    pitchedRoof?: number;
-    /** Printed solid volume must be < this fraction of the bbox volume (hollow). */
-    hollowBelow?: number;
+  volumes?: [number, number];
+  bbox?: {
+    x?: [number, number];
+    y?: [number, number];
+    z?: [number, number];
+  };
+  minTriangles?: number;
+  /** Top-band XY footprint must be < this fraction of the base footprint. */
+  pitchedRoof?: number;
+  /** Printed solid volume must be < this fraction of the bbox volume (hollow). */
+  hollowBelow?: number;
 }
 declare function createCadDesignAdapter(): BenchmarkAdapter;
-
-export { type CadSpec, createCadDesignAdapter };
+//#endregion
+export { CadSpec, createCadDesignAdapter };
+//# sourceMappingURL=cad-design.d.ts.map