@ai-sdk-tool/eval 0.1.1

package/dist/index.js

@@ -0,0 +1,834 @@
+// src/reporters/console.ts
+var colors = {
+  reset: "\x1B[0m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  magenta: "\x1B[35m"
+};
+function printResult(result) {
+  const { model, benchmark, result: benchmarkResult } = result;
+  const status = benchmarkResult.success ? `${colors.green}\u2714 SUCCESS${colors.reset}` : `${colors.red}\u2716 FAILURE${colors.reset}`;
+  console.log(
+    `
+ ${colors.cyan}[${model}]${colors.reset} - ${colors.magenta}${benchmark}${colors.reset}`
+  );
+  console.log(
+    `  \u2514 ${status} | Score: ${colors.yellow}${benchmarkResult.score.toFixed(2)}${colors.reset}`
+  );
+  const metrics = Object.entries(benchmarkResult.metrics);
+  if (metrics.length > 0) {
+    console.log("    Metrics:");
+    for (const [key, value] of metrics) {
+      console.log(`      - ${key}: ${value}`);
+    }
+  }
+  if (benchmarkResult.error) {
+    console.log(
+      `    ${colors.red}Error: ${benchmarkResult.error.message}${colors.reset}`
+    );
+  }
+}
+function consoleReporter(results) {
+  console.log("\n--- \u{1F4CA} Evaluation Report ---");
+  for (const result of results) {
+    printResult(result);
+  }
+  console.log("\n---------------------------\n");
+}
+
+// src/reporters/json.ts
+function jsonReporter(results) {
+  const serializableResults = results.map((r) => ({
+    ...r,
+    result: {
+      ...r.result,
+      error: r.result.error?.message
+    }
+  }));
+  console.log(JSON.stringify(serializableResults, null, 2));
+}
+
+// src/reporters/index.ts
+var reporters = {
+  console: consoleReporter,
+  json: jsonReporter
+};
+
+// src/evaluate.ts
+async function runSingleBenchmark(model, benchmark) {
+  const modelId = typeof model === "object" && model !== null && "modelId" in model && typeof model.modelId === "string" ? model.modelId : "unknown-model";
+  try {
+    console.log(`[${modelId}] Running benchmark: ${benchmark.name}...`);
+    const result = await benchmark.run(model);
+    console.log(
+      `[${modelId}] Finished benchmark: ${benchmark.name}. Score: ${result.score}`
+    );
+    return {
+      model: modelId,
+      benchmark: benchmark.name,
+      result
+    };
+  } catch (error) {
+    console.error(
+      `[${modelId}] Error running benchmark: ${benchmark.name}`,
+      error
+    );
+    return {
+      model: modelId,
+      benchmark: benchmark.name,
+      result: {
+        score: 0,
+        success: false,
+        metrics: {},
+        error: error instanceof Error ? error : new Error(String(error))
+      }
+    };
+  }
+}
+async function evaluate(options) {
+  const { models, benchmarks, reporter = "console" } = options;
+  const modelsArray = Array.isArray(models) ? models : [models];
+  const allResults = [];
+  for (const model of modelsArray) {
+    for (const benchmark of benchmarks) {
+      const evaluationResult = await runSingleBenchmark(model, benchmark);
+      allResults.push(evaluationResult);
+    }
+  }
+  const report = reporters[reporter];
+  if (report) {
+    report(allResults);
+  } else {
+    console.warn(`Unknown reporter: '${reporter}'. Defaulting to console.`);
+    reporters.console(allResults);
+  }
+  return allResults;
+}
+
+// src/benchmarks/json-generation.ts
+import { generateText } from "ai";
+import Ajv from "ajv";
+import { promises as fs2 } from "fs";
+import path2 from "path";
+
+// src/utils/paths.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+function resolveDataDir(fromModuleUrl) {
+  const moduleUrl = fromModuleUrl;
+  const override = process.env.BFCL_DATA_DIR;
+  if (override && override.trim().length > 0) {
+    return override;
+  }
+  try {
+    const baseForRequireEntry = typeof moduleUrl === "string" && moduleUrl || path.join(process.cwd(), "package.json");
+    const requireFromEntry = createRequire(baseForRequireEntry);
+    const entryPath = requireFromEntry.resolve("@ai-sdk-tool/eval");
+    const entryDir = path.dirname(entryPath);
+    const guessPkgRoot = fs.existsSync(path.join(entryDir, "..")) ? path.resolve(entryDir, "..") : entryDir;
+    const dataAtRoot = path.join(guessPkgRoot, "data");
+    if (fs.existsSync(dataAtRoot)) return dataAtRoot;
+  } catch {
+  }
+  try {
+    const baseForRequire = typeof moduleUrl === "string" && moduleUrl || path.join(process.cwd(), "package.json");
+    const require2 = createRequire(baseForRequire);
+    const pkgJsonPath = require2.resolve("@ai-sdk-tool/eval/package.json");
+    const pkgDir = path.dirname(pkgJsonPath);
+    const dataAtPkg = path.join(pkgDir, "data");
+    if (fs.existsSync(dataAtPkg)) return dataAtPkg;
+  } catch {
+  }
+  let startDir;
+  if (moduleUrl) {
+    try {
+      startDir = path.dirname(fileURLToPath(moduleUrl));
+    } catch {
+      startDir = process.cwd();
+    }
+  } else {
+    startDir = process.cwd();
+  }
+  let dir = startDir;
+  for (let i = 0; i < 6; i++) {
+    const dataCandidate = path.join(dir, "data");
+    if (fs.existsSync(dataCandidate)) return dataCandidate;
+    const parent = path.resolve(dir, "..");
+    if (parent === dir) break;
+    dir = parent;
+  }
+  const pkgRoot = path.resolve(startDir, "..", "..");
+  return path.join(pkgRoot, "data");
+}
+
+// src/benchmarks/json-generation.ts
+function extractFirstJsonBlock(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+  }
+  const fenceMatch = text.match(/```json\s*([\s\S]*?)```/i) || text.match(/```\s*([\s\S]*?)```/i);
+  if (fenceMatch) {
+    const inner = fenceMatch[1].trim();
+    try {
+      return JSON.parse(inner);
+    } catch {
+    }
+  }
+  const startIdxObj = text.indexOf("{");
+  const startIdxArr = text.indexOf("[");
+  const start = [startIdxObj, startIdxArr].filter((i) => i >= 0).sort((a, b) => a - b)[0];
+  if (start === void 0) return void 0;
+  const open = text[start] === "{" ? "{" : "[";
+  const close = open === "{" ? "}" : "]";
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    const ch = text[i];
+    if (ch === open) depth++;
+    else if (ch === close) depth--;
+    if (depth === 0) {
+      const candidate = text.slice(start, i + 1);
+      try {
+        return JSON.parse(candidate);
+      } catch {
+      }
+      break;
+    }
+  }
+  return void 0;
+}
+function subsetMatch(expected, actual) {
+  if (expected === null || typeof expected !== "object") {
+    return expected === actual;
+  }
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return false;
+    for (let i = 0; i < expected.length; i++) {
+      if (!subsetMatch(expected[i], actual[i])) return false;
+    }
+    return true;
+  }
+  if (actual === null || typeof actual !== "object") return false;
+  const eObj = expected;
+  const aObj = actual;
+  for (const key of Object.keys(eObj)) {
+    if (!subsetMatch(eObj[key], aObj[key])) return false;
+  }
+  return true;
+}
+var jsonGenerationBenchmark = {
+  name: "json-generation",
+  version: "2.1.0",
+  description: "Evaluates schema-compliant JSON generation from natural language using JSON Schema prompts.",
+  async run(model) {
+    const logs = [];
+    const ajv = new Ajv({ allErrors: true, strict: false });
+    let schemaValidCount = 0;
+    let valueMatchCount = 0;
+    let correctCount = 0;
+    let tests = [];
+    const expectedMap = /* @__PURE__ */ new Map();
+    try {
+      const dataDir = resolveDataDir();
+      const testsJsonl = await fs2.readFile(
+        path2.join(dataDir, "json_generation_tests.jsonl"),
+        "utf-8"
+      );
+      const expectedJsonl = await fs2.readFile(
+        path2.join(dataDir, "json_generation_expected.jsonl"),
+        "utf-8"
+      );
+      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+      const expecteds = expectedJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+      for (const r of expecteds) expectedMap.set(r.id, r);
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return {
+        score: 0,
+        success: false,
+        metrics: {},
+        logs: [`[FATAL] Failed to load json-generation datasets: ${msg}`],
+        error: e
+      };
+    }
+    for (const tc of tests) {
+      try {
+        const schemaStr = JSON.stringify(tc.schema, null, 2);
+        const messages = [
+          {
+            role: "system",
+            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
+          },
+          {
+            role: "user",
+            content: [
+              "Generate a JSON object that reflects the following facts.",
+              "JSON Schema:",
+              schemaStr,
+              "Facts:",
+              tc.promptFacts,
+              "Output must be a single JSON only, with no additional text."
+            ].join("\n\n")
+          }
+        ];
+        const { text } = await generateText({ model, messages });
+        let parsed;
+        try {
+          parsed = extractFirstJsonBlock(text);
+        } catch {
+        }
+        if (parsed === void 0) {
+          logs.push(`[FAIL] ${tc.id}: Unable to parse JSON from model output.`);
+          continue;
+        }
+        const validate = ajv.compile(tc.schema);
+        const valid = validate(parsed);
+        if (valid) schemaValidCount++;
+        else
+          logs.push(
+            `[INFO] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
+          );
+        const expectedRec = expectedMap.get(tc.id);
+        if (!expectedRec) {
+          logs.push(
+            `[WARN] ${tc.id}: No expected record found. Skipping value match.`
+          );
+        }
+        const valuesOk = expectedRec ? subsetMatch(expectedRec.expected, parsed) : false;
+        if (valuesOk) valueMatchCount++;
+        if (valid && valuesOk) {
+          correctCount++;
+          logs.push(`[PASS] ${tc.id}`);
+        } else {
+          logs.push(
+            `[FAIL] ${tc.id}: schemaValid=${valid}, valuesOk=${valuesOk}. Output=${JSON.stringify(
+              parsed
+            )}`
+          );
+        }
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        logs.push(`[ERROR] ${tc.id}: ${msg}`);
+      }
+    }
+    const total = tests.length;
+    const score = correctCount / total;
+    return {
+      score,
+      success: score >= 0.8,
+      metrics: {
+        total_cases: total,
+        correct_count: correctCount,
+        schema_valid_count: schemaValidCount,
+        value_match_count: valueMatchCount,
+        accuracy: score
+      },
+      logs
+    };
+  }
+};
+var jsonGenerationSchemaOnlyBenchmark = {
+  name: "json-generation-schema-only",
+  version: "1.0.1",
+  description: "Evaluates whether model outputs strictly conform to the provided JSON Schema (structure only).",
+  async run(model) {
+    const logs = [];
+    const ajv = new Ajv({ allErrors: true, strict: false });
+    let tests = [];
+    try {
+      const dataDir = resolveDataDir();
+      const testsJsonl = await fs2.readFile(
+        path2.join(dataDir, "json_generation_tests.jsonl"),
+        "utf-8"
+      );
+      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return {
+        score: 0,
+        success: false,
+        metrics: {},
+        logs: [`[FATAL] Failed to load schema-only tests: ${msg}`],
+        error: e
+      };
+    }
+    let schemaValidCount = 0;
+    for (const tc of tests) {
+      try {
+        const schemaStr = JSON.stringify(tc.schema, null, 2);
+        const messages = [
+          {
+            role: "system",
+            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
+          },
+          {
+            role: "user",
+            content: [
+              "Generate a JSON object that reflects the following facts.",
+              "JSON Schema:",
+              schemaStr,
+              "Facts:",
+              tc.promptFacts,
+              "Output must be a single JSON only, with no additional text."
+            ].join("\n\n")
+          }
+        ];
+        const { text } = await generateText({ model, messages });
+        let parsed;
+        try {
+          parsed = extractFirstJsonBlock(text);
+        } catch {
+        }
+        if (parsed === void 0) {
+          logs.push(`[FAIL] ${tc.id}: Could not parse JSON from model output.`);
+          continue;
+        }
+        const validate = ajv.compile(tc.schema);
+        const valid = validate(parsed);
+        if (valid) {
+          schemaValidCount++;
+          logs.push(`[PASS] ${tc.id}`);
+        } else {
+          logs.push(
+            `[FAIL] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
+          );
+        }
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        logs.push(`[ERROR] ${tc.id}: ${msg}`);
+      }
+    }
+    const total = tests.length;
+    const score = total > 0 ? schemaValidCount / total : 0;
+    return {
+      score,
+      success: score >= 0.8,
+      metrics: {
+        total_cases: total,
+        schema_valid_count: schemaValidCount,
+        accuracy: score
+      },
+      logs
+    };
+  }
+};
+
+// src/benchmarks/bfcl.ts
+import { generateText as generateText2, jsonSchema } from "ai";
+import { promises as fs3 } from "fs";
+import path3 from "path";
+
+// src/benchmarks/bfcl/ast-checker.ts
+function standardizeString(input) {
+  if (typeof input !== "string") return input;
+  const regex = /[ ,./\\-_*^]/g;
+  return input.replace(regex, "").toLowerCase().replace(/'/g, '"');
+}
+function checkStringValue(param, modelValue, possibleAnswers) {
+  const standardizedModelValue = standardizeString(modelValue);
+  const standardizedPossibleAnswers = possibleAnswers.map(
+    (ans) => standardizeString(ans)
+  );
+  if (!standardizedPossibleAnswers.includes(standardizedModelValue)) {
+    return {
+      valid: false,
+      error: `Invalid value for parameter '${param}': '${modelValue}'. Expected one of ${possibleAnswers.join(", ")}.`,
+      error_type: "value_error:string"
+    };
+  }
+  return { valid: true };
+}
+function simpleFunctionChecker(funcDescription, modelToolCall, possibleAnswer) {
+  const modelArgs = modelToolCall.args;
+  const modelFuncName = modelToolCall.toolName;
+  const expectedFuncName = funcDescription.name;
+  const expectedParams = funcDescription.parameters.properties;
+  const requiredParams = funcDescription.parameters.required;
+  if (modelFuncName !== expectedFuncName) {
+    return {
+      valid: false,
+      error: `Function name '${modelFuncName}' does not match expected '${expectedFuncName}'.`,
+      error_type: "simple_function_checker:wrong_func_name"
+    };
+  }
+  const possibleAnswerParams = possibleAnswer[Object.keys(possibleAnswer)[0]];
+  for (const param of requiredParams) {
+    if (!(param in modelArgs)) {
+      return {
+        valid: false,
+        error: `Missing required parameter: '${param}'.`,
+        error_type: "simple_function_checker:missing_required"
+      };
+    }
+  }
+  for (const paramName in modelArgs) {
+    const modelValue = modelArgs[paramName];
+    if (!(paramName in expectedParams) || !(paramName in possibleAnswerParams)) {
+      return {
+        valid: false,
+        error: `Unexpected parameter: '${paramName}'.`,
+        error_type: "simple_function_checker:unexpected_param"
+      };
+    }
+    const possibleValues = possibleAnswerParams[paramName];
+    if (typeof modelValue === "string") {
+      const result = checkStringValue(paramName, modelValue, possibleValues);
+      if (!result.valid) return result;
+    } else if (Array.isArray(modelValue)) {
+      const modelValueStr = JSON.stringify(
+        modelValue.map((v) => standardizeString(v.toString())).sort()
+      );
+      const hasMatch = possibleValues.some(
+        (p) => JSON.stringify(
+          p.map((v) => standardizeString(v.toString())).sort()
+        ) === modelValueStr
+      );
+      if (!hasMatch) {
+        return {
+          valid: false,
+          error: `Invalid value for list parameter '${paramName}'.`,
+          error_type: "value_error:list"
+        };
+      }
+    } else {
+      if (!possibleValues.includes(modelValue)) {
+        return {
+          valid: false,
+          error: `Invalid value for parameter '${paramName}': got '${modelValue}', expected one of '${possibleValues}'.`,
+          error_type: "value_error:other"
+        };
+      }
+    }
+  }
+  for (const paramName in possibleAnswerParams) {
+    if (!(paramName in modelArgs) && !possibleAnswerParams[paramName].includes("")) {
+      return {
+        valid: false,
+        error: `Missing optional parameter '${paramName}' which was not marked as optional.`,
+        error_type: "simple_function_checker:missing_optional"
+      };
+    }
+  }
+  return { valid: true };
+}
+function parallelFunctionCheckerNoOrder(funcDescriptions, modelToolCalls, possibleAnswers) {
+  if (modelToolCalls.length !== possibleAnswers.length) {
+    return {
+      valid: false,
+      error: `Wrong number of functions. Expected ${possibleAnswers.length}, got ${modelToolCalls.length}.`,
+      error_type: "parallel_function_checker_no_order:wrong_count"
+    };
+  }
+  const matchedModelCallIndices = /* @__PURE__ */ new Set();
+  for (const possibleAnswer of possibleAnswers) {
+    const expectedFuncName = Object.keys(possibleAnswer)[0];
+    const funcDescription = funcDescriptions.find(
+      (f) => f.name === expectedFuncName
+    );
+    if (!funcDescription) {
+      return {
+        valid: false,
+        error: `Could not find function description for '${expectedFuncName}'.`,
+        error_type: "parallel_function_checker_no_order:missing_func_desc"
+      };
+    }
+    let foundMatch = false;
+    for (let i = 0; i < modelToolCalls.length; i++) {
+      if (matchedModelCallIndices.has(i)) continue;
+      const checkerResult = simpleFunctionChecker(
+        funcDescription,
+        modelToolCalls[i],
+        possibleAnswer
+      );
+      if (checkerResult.valid) {
+        matchedModelCallIndices.add(i);
+        foundMatch = true;
+        break;
+      }
+    }
+    if (!foundMatch) {
+      return {
+        valid: false,
+        error: `Could not find a matching function call for '${expectedFuncName}'.`,
+        error_type: "parallel_function_checker_no_order:cannot_find_match"
+      };
+    }
+  }
+  return { valid: true };
+}
+function multipleFunctionChecker(funcDescriptions, modelToolCalls, possibleAnswers) {
+  if (modelToolCalls.length !== possibleAnswers.length) {
+    return {
+      valid: false,
+      error: `Wrong number of functions. Expected ${possibleAnswers.length}, got ${modelToolCalls.length}.`,
+      error_type: "multiple_function_checker:wrong_count"
+    };
+  }
+  const expectedFuncName = Object.keys(possibleAnswers[0])[0];
+  const funcDescription = funcDescriptions.find(
+    (f) => f.name === expectedFuncName
+  );
+  if (!funcDescription) {
+    return {
+      valid: false,
+      error: `Could not find function description for '${expectedFuncName}'.`,
+      error_type: "multiple_function_checker:missing_func_desc"
+    };
+  }
+  return simpleFunctionChecker(
+    funcDescription,
+    modelToolCalls[0],
+    possibleAnswers[0]
+  );
+}
+
+// src/benchmarks/bfcl.ts
+function check(testCase, modelOutput, possibleAnswer) {
+  const category = testCase.id.split("_")[0];
+  try {
+    if (category === "simple") {
+      if (!modelOutput || modelOutput.length !== 1) {
+        return {
+          valid: false,
+          error: `Expected 1 function call, but got ${modelOutput?.length ?? 0}.`
+        };
+      }
+      return simpleFunctionChecker(
+        testCase.function[0],
+        modelOutput[0],
+        possibleAnswer.ground_truth[0]
+      );
+    } else if (category === "parallel") {
+      return parallelFunctionCheckerNoOrder(
+        testCase.function,
+        modelOutput,
+        possibleAnswer.ground_truth
+      );
+    } else if (category === "multiple") {
+      return multipleFunctionChecker(
+        testCase.function,
+        modelOutput,
+        possibleAnswer.ground_truth
+      );
+    } else if (category.includes("parallel-multiple")) {
+      return parallelFunctionCheckerNoOrder(
+        testCase.function,
+        modelOutput,
+        possibleAnswer.ground_truth
+      );
+    }
+    return { valid: true };
+  } catch (e) {
+    return { valid: false, error: `Checker Error: ${e.message}` };
+  }
+}
+function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
+  return {
+    name,
+    version: "1.0.0",
+    description,
+    async run(model) {
+      const logs = [];
+      let correctCount = 0;
+      let testCases = [];
+      try {
+        const dataPath = resolveDataDir();
+        logs.push(`[INFO] Using data dir: ${dataPath}`);
+        const testCasesJson = await fs3.readFile(
+          path3.join(dataPath, testDataFile),
+          "utf-8"
+        );
+        const possibleAnswersJson = await fs3.readFile(
+          path3.join(dataPath, answerDataFile),
+          "utf-8"
+        );
+        testCases = testCasesJson.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+        const possibleAnswers = possibleAnswersJson.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+        const possibleAnswersMap = new Map(
+          possibleAnswers.map((ans) => [ans.id, ans])
+        );
+        const limitEnv = process.env.BFCL_LIMIT;
+        const limit = limitEnv ? Number(limitEnv) : void 0;
+        if (limit && Number.isFinite(limit) && limit > 0) {
+          testCases = testCases.slice(0, limit);
+          logs.push(
+            `[INFO] Limiting test cases to ${limit} due to BFCL_LIMIT.`
+          );
+        }
+        const fixSchema = (schema) => {
+          if (!schema || typeof schema !== "object") return schema;
+          const copy = Array.isArray(schema) ? schema.map((v) => fixSchema(v)) : { ...schema };
+          if (copy.type) {
+            if (copy.type === "dict") copy.type = "object";
+            if (copy.type === "integer" || copy.type === "float")
+              copy.type = "number";
+          }
+          if (copy.properties && typeof copy.properties === "object") {
+            for (const k of Object.keys(copy.properties)) {
+              copy.properties[k] = fixSchema(copy.properties[k]);
+            }
+          }
+          if (copy.items) copy.items = fixSchema(copy.items);
+          return copy;
+        };
+        for (const testCase of testCases) {
+          const { function: tools, question: messages } = testCase;
+          try {
+            const flatMessages = Array.isArray(messages) && messages.some((m) => Array.isArray(m)) ? messages.flat(1) : messages;
+            const nameMap = /* @__PURE__ */ new Map();
+            const sanitizeName = (name2) => {
+              const s = name2.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64);
+              return s.length > 0 ? s : "tool";
+            };
+            const transformedTools = tools.map((t) => {
+              const fixed = fixSchema(t.parameters);
+              const inputSchema = fixed && typeof fixed === "object" && fixed.type === "object" ? fixed : { type: "object", properties: {} };
+              const sanitized = sanitizeName(t.name);
+              nameMap.set(sanitized, t.name);
+              return {
+                type: "function",
+                name: sanitized,
+                description: t.description,
+                // Mark as JSON schema explicitly to prevent Zod parsing
+                inputSchema: jsonSchema(inputSchema)
+              };
+            });
+            try {
+              const firstTool = transformedTools[0];
+              const schemaType = firstTool?.inputSchema?.type ?? firstTool?.inputSchema?.jsonSchema?.type;
+              logs.push(
+                `[DEBUG] ${testCase.id}: firstTool=${JSON.stringify(firstTool)}, schemaType=${schemaType}`
+              );
+            } catch (e) {
+              logs.push(
+                `[DEBUG] ${testCase.id}: failed to introspect tools: ${e.message}`
+              );
+            }
+            const { toolCalls, text, finishReason } = await generateText2({
+              model,
+              messages: flatMessages,
+              tools: transformedTools,
+              toolChoice: "required"
+            });
+            try {
+              logs.push(
+                `[DEBUG] ${testCase.id}: rawToolCalls=${JSON.stringify(toolCalls)}, finishReason=${finishReason}, text=${JSON.stringify(text)}`
+              );
+            } catch {
+              logs.push(
+                `[DEBUG] ${testCase.id}: failed to serialize toolCalls`
+              );
+            }
+            const possibleAnswer = possibleAnswersMap.get(testCase.id);
+            if (!possibleAnswer) {
+              throw new Error(`No possible answer for id: ${testCase.id}`);
+            }
+            const restoredCalls = (toolCalls || []).map((c) => {
+              const rawName = c.toolName ?? c.name;
+              const sanitizedFromIndex = typeof rawName === "string" && /^\d+$/.test(rawName) ? transformedTools[Number(rawName)]?.name ?? rawName : rawName;
+              const originalName = nameMap.get(sanitizedFromIndex) ?? sanitizedFromIndex;
+              const extractedArgs = c.args ?? c.arguments ?? c.input ?? c.params ?? c.parameters ?? void 0;
+              let parsedArgs = extractedArgs;
+              if (typeof parsedArgs === "string") {
+                try {
+                  parsedArgs = JSON.parse(parsedArgs);
+                } catch {
+                }
+              }
+              return {
+                ...c,
+                toolName: originalName,
+                name: originalName,
+                args: parsedArgs ?? {}
+              };
+            });
+            const checkerResult = check(
+              testCase,
+              restoredCalls,
+              possibleAnswer
+            );
+            if (checkerResult.valid) {
+              correctCount++;
+              logs.push(`[PASS] ${testCase.id}`);
+            } else {
+              logs.push(`[FAIL] ${testCase.id}: ${checkerResult.error}`);
+            }
+          } catch (e) {
+            logs.push(
+              `[ERROR] ${testCase.id}: Model generation failed: ${e?.message}`
+            );
+            if (e?.stack) {
+              logs.push(`[STACK] ${testCase.id}: ${e.stack}`);
+            }
+          }
+        }
+        if (testCases.length === 0) {
+          return {
+            score: 0,
+            success: false,
+            metrics: {},
+            logs: ["No test cases found."]
+          };
+        }
+        const score = correctCount / testCases.length;
+        return {
+          score,
+          success: score > 0.95,
+          // High success threshold as requested
+          metrics: {
+            correct_count: correctCount,
+            total_cases: testCases.length,
+            accuracy: score
+          },
+          logs
+        };
+      } catch (e) {
+        return {
+          score: 0,
+          success: false,
+          metrics: {},
+          error: e,
+          logs: [`[FATAL] Failed to run benchmark ${name}: ${e.message}`]
+        };
+      }
+    }
+  };
+}
+var bfclSimpleBenchmark = createBfclBenchmark(
+  "bfcl-simple",
+  "BFCL Simple Function Calling",
+  "BFCL_v3_simple.json",
+  "BFCL_v3_simple_possible_answer.json"
+);
+var bfclParallelBenchmark = createBfclBenchmark(
+  "bfcl-parallel",
+  "BFCL Parallel Function Calling",
+  "BFCL_v3_parallel.json",
+  "BFCL_v3_parallel_possible_answer.json"
+);
+var bfclMultipleBenchmark = createBfclBenchmark(
+  "bfcl-multiple",
+  "BFCL Multiple Function Calling",
+  "BFCL_v3_multiple.json",
+  "BFCL_v3_multiple_possible_answer.json"
+);
+var bfclParallelMultipleBenchmark = createBfclBenchmark(
+  "bfcl-parallel-multiple",
+  "BFCL Parallel & Multiple Function Calling",
+  "BFCL_v3_parallel_multiple.json",
+  "BFCL_v3_parallel_multiple_possible_answer.json"
+);
+export {
+  bfclMultipleBenchmark,
+  bfclParallelBenchmark,
+  bfclParallelMultipleBenchmark,
+  bfclSimpleBenchmark,
+  evaluate,
+  jsonGenerationBenchmark,
+  jsonGenerationSchemaOnlyBenchmark
+};
+//# sourceMappingURL=index.js.map