@ai-sdk-tool/eval 0.1.2 → 0.1.4

package/dist/index.js

@@ -5,14 +5,15 @@ var colors = {
   red: "\x1B[31m",
   yellow: "\x1B[33m",
   cyan: "\x1B[36m",
-  magenta: "\x1B[35m"
+  magenta: "\x1B[35m",
+  gray: "\x1B[90m"
 };
 function printResult(result) {
-  const { model, benchmark, result: benchmarkResult } = result;
+  const { model, modelKey, benchmark, result: benchmarkResult } = result;
   const status = benchmarkResult.success ? `${colors.green}\u2714 SUCCESS${colors.reset}` : `${colors.red}\u2716 FAILURE${colors.reset}`;
   console.log(
     `
- ${colors.cyan}[${model}]${colors.reset} - ${colors.magenta}${benchmark}${colors.reset}`
+ ${colors.cyan}[${model}]${colors.reset}${modelKey ? ` ${colors.gray}(${modelKey})${colors.reset}` : ""} - ${colors.magenta}${benchmark}${colors.reset}`
   );
   console.log(
     `  \u2514 ${status} | Score: ${colors.yellow}${benchmarkResult.score.toFixed(2)}${colors.reset}`
@@ -38,6 +39,186 @@ function consoleReporter(results) {
   console.log("\n---------------------------\n");
 }
 
+// src/reporters/console.debug.ts
+var colors2 = {
+  reset: "\x1B[0m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  magenta: "\x1B[35m",
+  gray: "\x1B[90m",
+  bold: "\x1B[1m",
+  underline: "\x1B[4m"
+};
+function colorizeDiffLine(line) {
+  if (line.startsWith("+")) return `${colors2.green}${line}${colors2.reset}`;
+  if (line.startsWith("-")) return `${colors2.red}${line}${colors2.reset}`;
+  if (line.startsWith("@"))
+    return `${colors2.cyan}${colors2.bold}${line}${colors2.reset}`;
+  return line;
+}
+function uniqueLines(lines) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const l of lines) {
+    if (seen.has(l)) continue;
+    seen.add(l);
+    out.push(l);
+  }
+  return out;
+}
+function suggestFixFromDiff(parsed) {
+  const suggestions = [];
+  const { error_type, expected, actual, diff } = parsed ?? {};
+  if (Array.isArray(diff) && diff.some((d) => String(d).includes("function name")) || Array.isArray(diff) && diff.some((d) => String(d).includes("missing function:"))) {
+    const expectedName = expected?.function;
+    const actualName = actual?.function;
+    if (expectedName && actualName && expectedName !== actualName) {
+      suggestions.push(
+        `Call the function '${expectedName}' instead of '${actualName}'.`
+      );
+    }
+    if (Array.isArray(expected?.functions)) {
+      suggestions.push(
+        `Ensure tool calls include: ${expected.functions.join(", ")}.`
+      );
+    }
+  }
+  if (Array.isArray(diff) && diff.some((d) => String(d).startsWith("- missing required param:"))) {
+    const missing = diff.filter((d) => String(d).startsWith("- missing required param:")).map((d) => String(d).replace("- missing required param: ", ""));
+    if (missing.length) {
+      suggestions.push(`Add required parameter(s): ${missing.join(", ")}.`);
+    }
+  }
+  if (Array.isArray(diff) && diff.some((d) => String(d).startsWith("+ unexpected param:"))) {
+    const extras = diff.filter((d) => String(d).startsWith("+ unexpected param:")).map((d) => String(d).replace("+ unexpected param: ", ""));
+    if (extras.length) {
+      suggestions.push(`Remove unexpected parameter(s): ${extras.join(", ")}.`);
+    }
+  }
+  if (Array.isArray(diff) && diff.some((d) => String(d).startsWith("@@ param "))) {
+    const targets = diff.filter((d) => String(d).startsWith("@@ param ")).map((d) => String(d).replace("@@ param ", ""));
+    for (const param of targets) {
+      const allowedLine = diff.find(
+        (d) => String(d).startsWith("- expected one of:")
+      );
+      if (allowedLine) {
+        const allowed = allowedLine.replace("- expected one of: ", "");
+        suggestions.push(`Set '${param}' to one of: ${allowed}.`);
+      } else {
+        suggestions.push(`Adjust '${param}' to an allowed value.`);
+      }
+    }
+  }
+  if (suggestions.length === 0 && typeof error_type === "string") {
+    if (error_type.includes("missing_required")) {
+      suggestions.push(
+        "Add all required parameters defined by the tool schema."
+      );
+    } else if (error_type.includes("unexpected_param")) {
+      suggestions.push("Remove parameters not present in the tool schema.");
+    } else if (error_type.includes("wrong_count")) {
+      suggestions.push(
+        "Adjust the number of tool calls to match expected count."
+      );
+    } else if (error_type.includes("wrong_func_name")) {
+      suggestions.push("Use the exact expected function name from the schema.");
+    } else if (error_type.includes("value_error")) {
+      suggestions.push("Choose a value from the allowed options.");
+    }
+  }
+  return uniqueLines(suggestions);
+}
+function consoleDebugReporter(results) {
+  console.log("\n--- \u{1F4CA} Evaluation Report (debug) ---");
+  for (const r of results) {
+    const { model, modelKey, benchmark, result } = r;
+    const status = result.success ? `${colors2.green}\u2714 SUCCESS${colors2.reset}` : `${colors2.red}\u2716 FAILURE${colors2.reset}`;
+    console.log(
+      `
+ ${colors2.cyan}[${model}]${colors2.reset}${modelKey ? ` ${colors2.gray}(${modelKey})${colors2.reset}` : ""} - ${colors2.magenta}${benchmark}${colors2.reset}`
+    );
+    console.log(
+      `  \u2514 ${status} | Score: ${colors2.yellow}${result.score.toFixed(2)}${colors2.reset}`
+    );
+    const metrics = Object.entries(result.metrics);
+    if (metrics.length > 0) {
+      console.log("    Metrics:");
+      for (const [k, v] of metrics) console.log(`      - ${k}: ${v}`);
+    }
+    if (result.logs && result.logs.length) {
+      const failLogs = result.logs.filter(
+        (l) => l.startsWith("[FAIL]") || l.startsWith("[ERROR]") || l.startsWith("[FATAL]") || l.startsWith("[STACK]") || l.startsWith("[DEBUG-FAIL]")
+      );
+      const hasFails = failLogs.length > 0;
+      if (hasFails) {
+        console.log(`    ${colors2.bold}Failure details:${colors2.reset}`);
+        const debugIds = /* @__PURE__ */ new Set();
+        for (const l of failLogs) {
+          if (l.startsWith("[DEBUG-FAIL]")) {
+            try {
+              const parsed = JSON.parse(l.replace(/^\[DEBUG-FAIL\] /, ""));
+              if (parsed?.id) debugIds.add(String(parsed.id));
+            } catch {
+            }
+          }
+        }
+        for (const line of failLogs) {
+          if (line.startsWith("[FAIL]")) {
+            const m = line.match(/^\[FAIL\]\s+([^:]+):/);
+            const failId = m?.[1];
+            if (failId && debugIds.has(failId)) continue;
+            console.log(`      ${colors2.red}${line}${colors2.reset}`);
+          } else if (line.startsWith("[ERROR]") || line.startsWith("[FATAL]")) {
+            console.log(`      ${colors2.yellow}${line}${colors2.reset}`);
+          } else if (line.startsWith("[STACK]")) {
+            console.log(`      ${colors2.gray}${line}${colors2.reset}`);
+          } else if (line.startsWith("[DEBUG-FAIL]")) {
+            const payload = line.replace(/^\[DEBUG-FAIL\] /, "");
+            try {
+              const parsed = JSON.parse(payload);
+              const { id, expected, actual, message, diff } = parsed;
+              console.log(
+                `      ${colors2.underline}${id}${colors2.reset} ${message ? "- " + message : ""}`
+              );
+              if (diff && Array.isArray(diff)) {
+                for (const dLine of diff)
+                  console.log("        " + colorizeDiffLine(dLine));
+              } else {
+                console.log("        expected:");
+                console.log(
+                  colors2.green + "          " + JSON.stringify(expected, null, 2).split("\n").join("\n          ") + colors2.reset
+                );
+                console.log("        actual:");
+                console.log(
+                  colors2.red + "          " + JSON.stringify(actual, null, 2).split("\n").join("\n          ") + colors2.reset
+                );
+              }
+              const suggestions = suggestFixFromDiff(parsed);
+              if (suggestions.length) {
+                console.log(
+                  `        ${colors2.bold}Suggested fix:${colors2.reset}`
+                );
+                for (const s of suggestions) console.log(`          \u2022 ${s}`);
+              }
+            } catch {
+              console.log(`      ${line}`);
+            }
+          }
+        }
+      } else {
+        const info = result.logs.filter(
+          (l) => l.startsWith("[INFO]") || l.startsWith("[PASS]")
+        );
+        for (const line of info)
+          console.log(`      ${colors2.gray}${line}${colors2.reset}`);
+      }
+    }
+  }
+  console.log("\n------------------------------------\n");
+}
+
 // src/reporters/json.ts
 function jsonReporter(results) {
   const serializableResults = results.map((r) => ({
@@ -53,30 +234,35 @@ function jsonReporter(results) {
 // src/reporters/index.ts
 var reporters = {
   console: consoleReporter,
-  json: jsonReporter
+  json: jsonReporter,
+  "console.debug": consoleDebugReporter
 };
 
 // src/evaluate.ts
-async function runSingleBenchmark(model, benchmark) {
+async function runSingleBenchmark(model, benchmark, modelKey) {
   const modelId = typeof model === "object" && model !== null && "modelId" in model && typeof model.modelId === "string" ? model.modelId : "unknown-model";
   try {
-    console.log(`[${modelId}] Running benchmark: ${benchmark.name}...`);
+    console.log(
+      `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Running benchmark: ${benchmark.name}...`
+    );
     const result = await benchmark.run(model);
     console.log(
-      `[${modelId}] Finished benchmark: ${benchmark.name}. Score: ${result.score}`
+      `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Finished benchmark: ${benchmark.name}. Score: ${result.score}`
     );
     return {
       model: modelId,
+      modelKey,
       benchmark: benchmark.name,
       result
     };
   } catch (error) {
     console.error(
-      `[${modelId}] Error running benchmark: ${benchmark.name}`,
+      `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Error running benchmark: ${benchmark.name}`,
       error
     );
     return {
       model: modelId,
+      modelKey,
       benchmark: benchmark.name,
       result: {
         score: 0,
@@ -89,11 +275,26 @@ async function runSingleBenchmark(model, benchmark) {
 }
 async function evaluate(options) {
   const { models, benchmarks, reporter = "console" } = options;
-  const modelsArray = Array.isArray(models) ? models : [models];
+  const modelEntries = [];
+  if (Array.isArray(models)) {
+    for (const m of models) modelEntries.push([void 0, m]);
+  } else if (typeof models === "object" && models !== null && "modelId" in models) {
+    modelEntries.push([void 0, models]);
+  } else {
+    for (const [key, m] of Object.entries(
+      models
+    )) {
+      modelEntries.push([key, m]);
+    }
+  }
   const allResults = [];
-  for (const model of modelsArray) {
+  for (const [modelKey, model] of modelEntries) {
     for (const benchmark of benchmarks) {
-      const evaluationResult = await runSingleBenchmark(model, benchmark);
+      const evaluationResult = await runSingleBenchmark(
+        model,
+        benchmark,
+        modelKey
+      );
       allResults.push(evaluationResult);
     }
   }
@@ -107,17 +308,16 @@ async function evaluate(options) {
   return allResults;
 }
 
-// src/benchmarks/json-generation.ts
-import { generateText } from "ai";
-import Ajv from "ajv";
+// src/benchmarks/bfcl.ts
+import { generateText, jsonSchema, tool } from "ai";
 import { promises as fs2 } from "fs";
 import path2 from "path";
 
 // src/utils/paths.ts
 import fs from "fs";
+import { createRequire } from "module";
 import path from "path";
 import { fileURLToPath } from "url";
-import { createRequire } from "module";
 function resolveDataDir(fromModuleUrl) {
   const moduleUrl = fromModuleUrl;
   const override = process.env.BFCL_DATA_DIR;
@@ -165,396 +365,193 @@ function resolveDataDir(fromModuleUrl) {
   return path.join(pkgRoot, "data");
 }
 
-// src/benchmarks/json-generation.ts
-function extractFirstJsonBlock(text) {
-  try {
-    return JSON.parse(text);
-  } catch {
+// src/benchmarks/bfcl/ast-checker.ts
+function standardizeString(input) {
+  if (typeof input !== "string") return input;
+  const regex = /[ ,./\\-_*^]/g;
+  return input.replace(regex, "").toLowerCase().replace(/'/g, '"');
+}
+function checkStringValue(param, modelValue, possibleAnswers) {
+  const standardizedModelValue = standardizeString(modelValue);
+  const standardizedPossibleAnswers = possibleAnswers.map(
+    (ans) => standardizeString(String(ans))
+  );
+  if (!standardizedPossibleAnswers.includes(standardizedModelValue)) {
+    return {
+      valid: false,
+      error: `Invalid value for parameter '${param}': ${JSON.stringify(
+        modelValue
+      )}. Expected one of ${JSON.stringify(possibleAnswers)}.`,
+      error_type: "value_error:string"
+    };
   }
-  const fenceMatch = text.match(/```json\s*([\s\S]*?)```/i) || text.match(/```\s*([\s\S]*?)```/i);
-  if (fenceMatch) {
-    const inner = fenceMatch[1].trim();
-    try {
-      return JSON.parse(inner);
-    } catch {
+  return { valid: true };
+}
+function simpleFunctionChecker(funcDescription, modelToolCall, possibleAnswer) {
+  const modelArgs = modelToolCall.args;
+  const modelFuncName = modelToolCall.toolName;
+  const expectedFuncName = funcDescription.name;
+  const expectedParams = funcDescription.parameters.properties;
+  const requiredParams = funcDescription.parameters.required;
+  if (modelFuncName !== expectedFuncName) {
+    return {
+      valid: false,
+      error: `Function name '${modelFuncName}' does not match expected '${expectedFuncName}'.`,
+      error_type: "simple_function_checker:wrong_func_name"
+    };
+  }
+  const possibleAnswerParams = possibleAnswer[Object.keys(possibleAnswer)[0]];
+  const argsObj = modelArgs && typeof modelArgs === "object" ? modelArgs : {};
+  for (const param of requiredParams) {
+    if (!(param in argsObj)) {
+      return {
+        valid: false,
+        error: `Missing required parameter: '${param}'.`,
+        error_type: "simple_function_checker:missing_required"
+      };
     }
   }
-  const startIdxObj = text.indexOf("{");
-  const startIdxArr = text.indexOf("[");
-  const start = [startIdxObj, startIdxArr].filter((i) => i >= 0).sort((a, b) => a - b)[0];
-  if (start === void 0) return void 0;
-  const open = text[start] === "{" ? "{" : "[";
-  const close = open === "{" ? "}" : "]";
-  let depth = 0;
-  for (let i = start; i < text.length; i++) {
-    const ch = text[i];
-    if (ch === open) depth++;
-    else if (ch === close) depth--;
-    if (depth === 0) {
-      const candidate = text.slice(start, i + 1);
-      try {
-        return JSON.parse(candidate);
-      } catch {
+  if (modelArgs && typeof modelArgs === "object") {
+    for (const paramName of Object.keys(argsObj)) {
+      const modelValue = argsObj[paramName];
+      if (!(paramName in expectedParams) || !(paramName in possibleAnswerParams)) {
+        return {
+          valid: false,
+          error: `Unexpected parameter: '${paramName}'.`,
+          error_type: "simple_function_checker:unexpected_param"
+        };
+      }
+      const possibleValues = possibleAnswerParams[paramName];
+      if (typeof modelValue === "string") {
+        const result = checkStringValue(
+          paramName,
+          modelValue,
+          possibleValues ?? []
+        );
+        if (!result.valid) return result;
+      } else if (Array.isArray(modelValue)) {
+        const modelValueStr = JSON.stringify(
+          modelValue.map((v) => standardizeString(String(v))).sort()
+        );
+        const hasMatch = Array.isArray(possibleValues) ? possibleValues.some((p) => {
+          if (!Array.isArray(p)) return false;
+          return JSON.stringify(
+            p.map((v) => standardizeString(String(v))).sort()
+          ) === modelValueStr;
+        }) : false;
+        if (!hasMatch) {
+          return {
+            valid: false,
+            error: `Invalid value for list parameter '${paramName}'. Got ${JSON.stringify(
+              modelValue
+            )}. Expected one of ${JSON.stringify(possibleValues)}.`,
+            error_type: "value_error:list"
+          };
+        }
+      } else {
+        const hasMatch = Array.isArray(possibleValues) ? possibleValues.some((possibleValue) => {
+          if (modelValue === possibleValue) return true;
+          if (typeof modelValue === "object" && modelValue !== null && typeof possibleValue === "object" && possibleValue !== null) {
+            try {
+              const normalizeObject = (obj) => {
+                if (Array.isArray(obj)) {
+                  return obj.map(normalizeObject);
+                }
+                if (obj && typeof obj === "object") {
+                  const normalized = {};
+                  for (const [key, value] of Object.entries(
+                    obj
+                  )) {
+                    if (Array.isArray(value) && value.length === 1 && (typeof value[0] !== "object" || value[0] === null)) {
+                      normalized[key] = value[0];
+                    } else {
+                      normalized[key] = normalizeObject(value);
+                    }
+                  }
+                  return normalized;
+                }
+                return obj;
+              };
+              const normalizedModel = normalizeObject(modelValue);
+              const normalizedPossible = normalizeObject(possibleValue);
+              return JSON.stringify(normalizedModel) === JSON.stringify(normalizedPossible);
+            } catch {
+              return false;
+            }
+          }
+          if (typeof modelValue === "number" && typeof possibleValue === "string") {
+            return modelValue.toString() === possibleValue;
+          }
+          if (typeof modelValue === "string" && typeof possibleValue === "number") {
+            return modelValue === possibleValue.toString();
+          }
+          return false;
+        }) : false;
+        if (!hasMatch) {
+          return {
+            valid: false,
+            error: `Invalid value for parameter '${paramName}'. Got ${JSON.stringify(
+              modelValue
+            )}. Expected one of ${JSON.stringify(possibleValues)}.`,
+            error_type: "value_error:other"
+          };
+        }
       }
-      break;
     }
   }
-  return void 0;
-}
-function subsetMatch(expected, actual) {
-  if (expected === null || typeof expected !== "object") {
-    return expected === actual;
-  }
-  if (Array.isArray(expected)) {
-    if (!Array.isArray(actual)) return false;
-    for (let i = 0; i < expected.length; i++) {
-      if (!subsetMatch(expected[i], actual[i])) return false;
+  for (const paramName in possibleAnswerParams) {
+    const val = possibleAnswerParams[paramName];
+    const isOptional = Array.isArray(val) && val.includes("");
+    if (!(paramName in argsObj) && !isOptional) {
+      return {
+        valid: false,
+        error: `Missing optional parameter '${paramName}' which was not marked as optional.`,
+        error_type: "simple_function_checker:missing_optional"
+      };
     }
-    return true;
-  }
-  if (actual === null || typeof actual !== "object") return false;
-  const eObj = expected;
-  const aObj = actual;
-  for (const key of Object.keys(eObj)) {
-    if (!subsetMatch(eObj[key], aObj[key])) return false;
   }
-  return true;
+  return { valid: true };
 }
-var jsonGenerationBenchmark = {
-  name: "json-generation",
-  version: "2.1.0",
-  description: "Evaluates schema-compliant JSON generation from natural language using JSON Schema prompts.",
-  async run(model) {
-    const logs = [];
-    const ajv = new Ajv({ allErrors: true, strict: false });
-    let schemaValidCount = 0;
-    let valueMatchCount = 0;
-    let correctCount = 0;
-    let tests = [];
-    const expectedMap = /* @__PURE__ */ new Map();
-    try {
-      const dataDir = resolveDataDir();
-      const testsJsonl = await fs2.readFile(
-        path2.join(dataDir, "json_generation_tests.jsonl"),
-        "utf-8"
-      );
-      const expectedJsonl = await fs2.readFile(
-        path2.join(dataDir, "json_generation_expected.jsonl"),
-        "utf-8"
+function parallelFunctionCheckerNoOrder(funcDescriptions, modelToolCalls, possibleAnswers) {
+  if (modelToolCalls.length !== possibleAnswers.length) {
+    return {
+      valid: false,
+      error: `Wrong number of functions. Expected ${possibleAnswers.length}, got ${modelToolCalls.length}.`,
+      error_type: "parallel_function_checker_no_order:wrong_count"
+    };
+  }
+  const matchedModelCallIndices = /* @__PURE__ */ new Set();
+  for (const possibleAnswer of possibleAnswers) {
+    const expectedFuncName = Object.keys(possibleAnswer)[0];
+    const funcDescription = funcDescriptions.find(
+      (f) => f.name === expectedFuncName
+    );
+    if (!funcDescription) {
+      return {
+        valid: false,
+        error: `Could not find function description for '${expectedFuncName}'.`,
+        error_type: "parallel_function_checker_no_order:missing_func_desc"
+      };
+    }
+    let foundMatch = false;
+    for (let i = 0; i < modelToolCalls.length; i++) {
+      if (matchedModelCallIndices.has(i)) continue;
+      const checkerResult = simpleFunctionChecker(
+        funcDescription,
+        modelToolCalls[i],
+        possibleAnswer
       );
-      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
-      const expecteds = expectedJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
-      for (const r of expecteds) expectedMap.set(r.id, r);
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
+      if (checkerResult.valid) {
+        matchedModelCallIndices.add(i);
+        foundMatch = true;
+        break;
+      }
+    }
+    if (!foundMatch) {
       return {
-        score: 0,
-        success: false,
-        metrics: {},
-        logs: [`[FATAL] Failed to load json-generation datasets: ${msg}`],
-        error: e
-      };
-    }
-    for (const tc of tests) {
-      try {
-        const schemaStr = JSON.stringify(tc.schema, null, 2);
-        const messages = [
-          {
-            role: "system",
-            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
-          },
-          {
-            role: "user",
-            content: [
-              "Generate a JSON object that reflects the following facts.",
-              "JSON Schema:",
-              schemaStr,
-              "Facts:",
-              tc.promptFacts,
-              "Output must be a single JSON only, with no additional text."
-            ].join("\n\n")
-          }
-        ];
-        const { text } = await generateText({ model, messages });
-        let parsed;
-        try {
-          parsed = extractFirstJsonBlock(text);
-        } catch {
-        }
-        if (parsed === void 0) {
-          logs.push(`[FAIL] ${tc.id}: Unable to parse JSON from model output.`);
-          continue;
-        }
-        const validate = ajv.compile(tc.schema);
-        const valid = validate(parsed);
-        if (valid) schemaValidCount++;
-        else
-          logs.push(
-            `[INFO] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
-          );
-        const expectedRec = expectedMap.get(tc.id);
-        if (!expectedRec) {
-          logs.push(
-            `[WARN] ${tc.id}: No expected record found. Skipping value match.`
-          );
-        }
-        const valuesOk = expectedRec ? subsetMatch(expectedRec.expected, parsed) : false;
-        if (valuesOk) valueMatchCount++;
-        if (valid && valuesOk) {
-          correctCount++;
-          logs.push(`[PASS] ${tc.id}`);
-        } else {
-          logs.push(
-            `[FAIL] ${tc.id}: schemaValid=${valid}, valuesOk=${valuesOk}. Output=${JSON.stringify(
-              parsed
-            )}`
-          );
-        }
-      } catch (e) {
-        const msg = e instanceof Error ? e.message : String(e);
-        logs.push(`[ERROR] ${tc.id}: ${msg}`);
-      }
-    }
-    const total = tests.length;
-    const score = correctCount / total;
-    return {
-      score,
-      success: score >= 0.8,
-      metrics: {
-        total_cases: total,
-        correct_count: correctCount,
-        schema_valid_count: schemaValidCount,
-        value_match_count: valueMatchCount,
-        accuracy: score
-      },
-      logs
-    };
-  }
-};
-var jsonGenerationSchemaOnlyBenchmark = {
-  name: "json-generation-schema-only",
-  version: "1.0.1",
-  description: "Evaluates whether model outputs strictly conform to the provided JSON Schema (structure only).",
-  async run(model) {
-    const logs = [];
-    const ajv = new Ajv({ allErrors: true, strict: false });
-    let tests = [];
-    try {
-      const dataDir = resolveDataDir();
-      const testsJsonl = await fs2.readFile(
-        path2.join(dataDir, "json_generation_tests.jsonl"),
-        "utf-8"
-      );
-      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      return {
-        score: 0,
-        success: false,
-        metrics: {},
-        logs: [`[FATAL] Failed to load schema-only tests: ${msg}`],
-        error: e
-      };
-    }
-    let schemaValidCount = 0;
-    for (const tc of tests) {
-      try {
-        const schemaStr = JSON.stringify(tc.schema, null, 2);
-        const messages = [
-          {
-            role: "system",
-            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
-          },
-          {
-            role: "user",
-            content: [
-              "Generate a JSON object that reflects the following facts.",
-              "JSON Schema:",
-              schemaStr,
-              "Facts:",
-              tc.promptFacts,
-              "Output must be a single JSON only, with no additional text."
-            ].join("\n\n")
-          }
-        ];
-        const { text } = await generateText({ model, messages });
-        let parsed;
-        try {
-          parsed = extractFirstJsonBlock(text);
-        } catch {
-        }
-        if (parsed === void 0) {
-          logs.push(`[FAIL] ${tc.id}: Could not parse JSON from model output.`);
-          continue;
-        }
-        const validate = ajv.compile(tc.schema);
-        const valid = validate(parsed);
-        if (valid) {
-          schemaValidCount++;
-          logs.push(`[PASS] ${tc.id}`);
-        } else {
-          logs.push(
-            `[FAIL] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
-          );
-        }
-      } catch (e) {
-        const msg = e instanceof Error ? e.message : String(e);
-        logs.push(`[ERROR] ${tc.id}: ${msg}`);
-      }
-    }
-    const total = tests.length;
-    const score = total > 0 ? schemaValidCount / total : 0;
-    return {
-      score,
-      success: score >= 0.8,
-      metrics: {
-        total_cases: total,
-        schema_valid_count: schemaValidCount,
-        accuracy: score
-      },
-      logs
-    };
-  }
-};
-
-// src/benchmarks/bfcl.ts
-import { generateText as generateText2, jsonSchema, tool } from "ai";
-import { promises as fs3 } from "fs";
-import path3 from "path";
-
-// src/benchmarks/bfcl/ast-checker.ts
-function standardizeString(input) {
-  if (typeof input !== "string") return input;
-  const regex = /[ ,./\\-_*^]/g;
-  return input.replace(regex, "").toLowerCase().replace(/'/g, '"');
-}
-function checkStringValue(param, modelValue, possibleAnswers) {
-  const standardizedModelValue = standardizeString(modelValue);
-  const standardizedPossibleAnswers = possibleAnswers.map(
-    (ans) => standardizeString(ans)
-  );
-  if (!standardizedPossibleAnswers.includes(standardizedModelValue)) {
-    return {
-      valid: false,
-      error: `Invalid value for parameter '${param}': '${modelValue}'. Expected one of ${possibleAnswers.join(", ")}.`,
-      error_type: "value_error:string"
-    };
-  }
-  return { valid: true };
-}
-function simpleFunctionChecker(funcDescription, modelToolCall, possibleAnswer) {
-  const modelArgs = modelToolCall.args;
-  const modelFuncName = modelToolCall.toolName;
-  const expectedFuncName = funcDescription.name;
-  const expectedParams = funcDescription.parameters.properties;
-  const requiredParams = funcDescription.parameters.required;
-  if (modelFuncName !== expectedFuncName) {
-    return {
-      valid: false,
-      error: `Function name '${modelFuncName}' does not match expected '${expectedFuncName}'.`,
-      error_type: "simple_function_checker:wrong_func_name"
-    };
-  }
-  const possibleAnswerParams = possibleAnswer[Object.keys(possibleAnswer)[0]];
-  for (const param of requiredParams) {
-    if (!(param in modelArgs)) {
-      return {
-        valid: false,
-        error: `Missing required parameter: '${param}'.`,
-        error_type: "simple_function_checker:missing_required"
-      };
-    }
-  }
-  for (const paramName in modelArgs) {
-    const modelValue = modelArgs[paramName];
-    if (!(paramName in expectedParams) || !(paramName in possibleAnswerParams)) {
-      return {
-        valid: false,
-        error: `Unexpected parameter: '${paramName}'.`,
-        error_type: "simple_function_checker:unexpected_param"
-      };
-    }
-    const possibleValues = possibleAnswerParams[paramName];
-    if (typeof modelValue === "string") {
-      const result = checkStringValue(paramName, modelValue, possibleValues);
-      if (!result.valid) return result;
-    } else if (Array.isArray(modelValue)) {
-      const modelValueStr = JSON.stringify(
-        modelValue.map((v) => standardizeString(v.toString())).sort()
-      );
-      const hasMatch = possibleValues.some(
-        (p) => JSON.stringify(
-          p.map((v) => standardizeString(v.toString())).sort()
-        ) === modelValueStr
-      );
-      if (!hasMatch) {
-        return {
-          valid: false,
-          error: `Invalid value for list parameter '${paramName}'.`,
-          error_type: "value_error:list"
-        };
-      }
-    } else {
-      if (!possibleValues.includes(modelValue)) {
-        return {
-          valid: false,
-          error: `Invalid value for parameter '${paramName}': got '${modelValue}', expected one of '${possibleValues}'.`,
-          error_type: "value_error:other"
-        };
-      }
-    }
-  }
-  for (const paramName in possibleAnswerParams) {
-    if (!(paramName in modelArgs) && !possibleAnswerParams[paramName].includes("")) {
-      return {
-        valid: false,
-        error: `Missing optional parameter '${paramName}' which was not marked as optional.`,
-        error_type: "simple_function_checker:missing_optional"
-      };
-    }
-  }
-  return { valid: true };
-}
-function parallelFunctionCheckerNoOrder(funcDescriptions, modelToolCalls, possibleAnswers) {
-  if (modelToolCalls.length !== possibleAnswers.length) {
-    return {
-      valid: false,
-      error: `Wrong number of functions. Expected ${possibleAnswers.length}, got ${modelToolCalls.length}.`,
-      error_type: "parallel_function_checker_no_order:wrong_count"
-    };
-  }
-  const matchedModelCallIndices = /* @__PURE__ */ new Set();
-  for (const possibleAnswer of possibleAnswers) {
-    const expectedFuncName = Object.keys(possibleAnswer)[0];
-    const funcDescription = funcDescriptions.find(
-      (f) => f.name === expectedFuncName
-    );
-    if (!funcDescription) {
-      return {
-        valid: false,
-        error: `Could not find function description for '${expectedFuncName}'.`,
-        error_type: "parallel_function_checker_no_order:missing_func_desc"
-      };
-    }
-    let foundMatch = false;
-    for (let i = 0; i < modelToolCalls.length; i++) {
-      if (matchedModelCallIndices.has(i)) continue;
-      const checkerResult = simpleFunctionChecker(
-        funcDescription,
-        modelToolCalls[i],
-        possibleAnswer
-      );
-      if (checkerResult.valid) {
-        matchedModelCallIndices.add(i);
-        foundMatch = true;
-        break;
-      }
-    }
-    if (!foundMatch) {
-      return {
-        valid: false,
-        error: `Could not find a matching function call for '${expectedFuncName}'.`,
-        error_type: "parallel_function_checker_no_order:cannot_find_match"
+        valid: false,
+        error: `Could not find a matching function call for '${expectedFuncName}'.`,
+        error_type: "parallel_function_checker_no_order:cannot_find_match"
       };
     }
   }
@@ -591,10 +588,11 @@ function check(testCase, modelOutput, possibleAnswer) {
   const category = testCase.id.split("_")[0];
   try {
     if (category === "simple") {
-      if (!modelOutput || modelOutput.length !== 1) {
+      if (!Array.isArray(modelOutput) || modelOutput.length !== 1) {
         return {
           valid: false,
-          error: `Expected 1 function call, but got ${modelOutput?.length ?? 0}.`
+          error: `Expected 1 function call, but got ${Array.isArray(modelOutput) ? modelOutput.length : 0}.`,
+          error_type: "simple:wrong_count"
         };
       }
       return simpleFunctionChecker(
@@ -623,7 +621,11 @@ function check(testCase, modelOutput, possibleAnswer) {
     }
     return { valid: true };
   } catch (e) {
-    return { valid: false, error: `Checker Error: ${e.message}` };
+    return {
+      valid: false,
+      error: `Checker Error: ${e.message}`,
+      error_type: "checker_error"
+    };
   }
 }
 function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
@@ -638,12 +640,12 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
       try {
         const dataPath = resolveDataDir();
         logs.push(`[INFO] Using data dir: ${dataPath}`);
-        const testCasesJson = await fs3.readFile(
-          path3.join(dataPath, testDataFile),
+        const testCasesJson = await fs2.readFile(
+          path2.join(dataPath, testDataFile),
           "utf-8"
         );
-        const possibleAnswersJson = await fs3.readFile(
-          path3.join(dataPath, answerDataFile),
+        const possibleAnswersJson = await fs2.readFile(
+          path2.join(dataPath, answerDataFile),
           "utf-8"
         );
         testCases = testCasesJson.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
@@ -660,22 +662,34 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
           );
         }
         const fixSchema = (schema) => {
-          if (!schema || typeof schema !== "object") return schema;
+          if (!schema || typeof schema !== "object")
+            return { type: "object", properties: {} };
           const copy = Array.isArray(schema) ? schema.map((v) => fixSchema(v)) : { ...schema };
-          if (copy.type) {
-            if (copy.type === "dict") copy.type = "object";
-            if (copy.type === "integer" || copy.type === "float")
-              copy.type = "number";
-          }
-          if (copy.properties && typeof copy.properties === "object") {
-            for (const k of Object.keys(copy.properties)) {
-              copy.properties[k] = fixSchema(copy.properties[k]);
+          if (!Array.isArray(copy)) {
+            if (copy.type) {
+              if (copy.type === "dict") copy.type = "object";
+              if (copy.type === "integer" || copy.type === "float")
+                copy.type = "number";
             }
+            if (copy.properties && typeof copy.properties === "object") {
+              for (const k of Object.keys(copy.properties)) {
+                copy.properties[k] = fixSchema(
+                  copy.properties[k]
+                );
+              }
+            }
+            if (copy.items) copy.items = fixSchema(copy.items);
+            return copy;
           }
-          if (copy.items) copy.items = fixSchema(copy.items);
           return copy;
         };
-        for (const testCase of testCases) {
+        const concurrencyEnv = process.env.BFCL_CONCURRENCY;
+        const concurrency = concurrencyEnv && Number.isFinite(Number(concurrencyEnv)) ? Math.max(1, Number(concurrencyEnv)) : 4;
+        logs.push(
+          `[INFO] Running ${testCases.length} test cases with concurrency=${concurrency}`
+        );
+        const runSingleCase = async (testCase) => {
+          const caseLogs = [];
           const { function: tools, question: messages } = testCase;
           try {
             const flatMessages = Array.isArray(messages) && messages.some((m) => Array.isArray(m)) ? messages.flat(1) : messages;
@@ -686,7 +700,8 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
             };
             const transformedTools = tools.map((t) => {
               const fixed = fixSchema(t.parameters);
-              const inputSchema = fixed && typeof fixed === "object" && fixed.type === "object" ? fixed : { type: "object", properties: {} };
+              const isObjectSchema = fixed && typeof fixed === "object" && fixed.type === "object";
+              const inputSchema = isObjectSchema ? fixed : { type: "object", properties: {} };
               const sanitized = sanitizeName(t.name);
               nameMap.set(sanitized, t.name);
               return {
@@ -708,26 +723,37 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
             try {
               const firstTool = transformedTools[0];
               const schemaType = firstTool?.inputSchema?.type ?? firstTool?.inputSchema?.jsonSchema?.type;
-              logs.push(
+              caseLogs.push(
                 `[DEBUG] ${testCase.id}: firstTool=${JSON.stringify(firstTool)}, schemaType=${schemaType}`
               );
             } catch (e) {
-              logs.push(
+              caseLogs.push(
                 `[DEBUG] ${testCase.id}: failed to introspect tools: ${e.message}`
               );
             }
-            const { toolCalls, text, finishReason } = await generateText2({
+            const { toolCalls, text, finishReason } = await generateText({
               model,
               messages: flatMessages,
               tools: toolsMap,
-              toolChoice: "auto"
+              toolChoice: "auto",
+              // Pass original schema information to middleware
+              providerOptions: {
+                toolCallMiddleware: {
+                  originalToolSchemas: Object.fromEntries(
+                    transformedTools.map((t) => [
+                      t.name,
+                      t.inputSchema
+                    ])
+                  )
+                }
+              }
             });
             try {
-              logs.push(
+              caseLogs.push(
                 `[DEBUG] ${testCase.id}: rawToolCalls=${JSON.stringify(toolCalls)}, finishReason=${finishReason}, text=${JSON.stringify(text)}`
               );
             } catch {
-              logs.push(
+              caseLogs.push(
                 `[DEBUG] ${testCase.id}: failed to serialize toolCalls`
               );
             }
@@ -760,20 +786,232 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
               possibleAnswer
             );
             if (checkerResult.valid) {
-              correctCount++;
-              logs.push(`[PASS] ${testCase.id}`);
+              caseLogs.push(`[PASS] ${testCase.id}`);
+              return { valid: true, logs: caseLogs };
             } else {
-              logs.push(`[FAIL] ${testCase.id}: ${checkerResult.error}`);
+              caseLogs.push(`[FAIL] ${testCase.id}: ${checkerResult.error}`);
+              try {
+                const category = testCase.id.split("_")[0];
+                const diff = [];
+                const summarizeArgs = (args) => {
+                  if (args == null) return args;
+                  if (typeof args !== "object") return args;
+                  return Object.keys(args).sort().reduce(
+                    (acc, k) => {
+                      acc[k] = args[k];
+                      return acc;
+                    },
+                    {}
+                  );
+                };
+                const expected = {};
+                const actual = {};
+                if (category === "simple") {
+                  const funcDesc = tools[0];
+                  const gt = possibleAnswer.ground_truth?.[0];
+                  const expectedFuncName = funcDesc?.name;
+                  const expectedParams = gt ? gt[Object.keys(gt)[0]] : void 0;
+                  const received = restoredCalls[0];
+                  const receivedName = received?.toolName ?? received?.name;
+                  const receivedArgs = summarizeArgs(received?.args);
+                  expected.function = expectedFuncName;
+                  expected.params = expectedParams;
+                  actual.function = receivedName;
+                  actual.args = receivedArgs;
+                  if (expectedFuncName !== receivedName) {
+                    diff.push(`@@ function name`);
+                    diff.push(`- ${expectedFuncName}`);
+                    diff.push(`+ ${receivedName}`);
+                  }
+                  if (expectedParams && receivedArgs && typeof receivedArgs === "object" && receivedArgs !== null) {
+                    const required = funcDesc?.parameters?.required ?? [];
+                    for (const req of required) {
+                      if (!(req in receivedArgs)) {
+                        diff.push(`- missing required param: ${req}`);
+                      }
+                    }
+                    for (const k of Object.keys(
+                      receivedArgs
+                    )) {
+                      if (!Object.prototype.hasOwnProperty.call(expectedParams, k)) {
+                        diff.push(`+ unexpected param: ${k}`);
+                      }
+                    }
+                    for (const k of Object.keys(
+                      receivedArgs
+                    )) {
+                      if (Object.prototype.hasOwnProperty.call(expectedParams, k)) {
+                        const allowed = expectedParams[k];
+                        const got = receivedArgs[k];
+                        const includes = Array.isArray(allowed) && allowed.some((v) => {
+                          try {
+                            if (Array.isArray(got)) {
+                              return JSON.stringify(
+                                got.map((x) => String(x)).sort()
+                              ) === JSON.stringify(
+                                v.map((x) => String(x)).sort()
+                              );
+                            }
+                          } catch {
+                          }
+                          return String(v).toLowerCase().replace(/\s+/g, "") === String(got).toLowerCase().replace(/\s+/g, "");
+                        });
+                        if (!includes) {
+                          diff.push(`@@ param ${k}`);
+                          diff.push(
+                            `- expected one of: ${JSON.stringify(allowed)}`
+                          );
+                          diff.push(`+ got: ${JSON.stringify(got)}`);
+                        }
+                      }
+                    }
+                  }
+                } else {
+                  const gtArr = possibleAnswer.ground_truth ?? [];
+                  const expectedNames = gtArr.map((g) => Object.keys(g)[0]);
+                  const actualNames = restoredCalls.map(
+                    (c) => c.toolName ?? c.name
+                  );
+                  expected.functions = expectedNames;
+                  actual.functions = actualNames;
+                  if (expectedNames.length !== actualNames.length) {
+                    diff.push(`@@ call count`);
+                    diff.push(`- expected ${expectedNames.length}`);
+                    diff.push(`+ got ${actualNames.length}`);
+                  }
+                  const missing = expectedNames.filter(
+                    (n) => !actualNames.includes(n)
+                  );
+                  const extra = actualNames.filter(
+                    (n) => !expectedNames.includes(n)
+                  );
+                  for (const m of missing)
+                    diff.push(`- missing function: ${m}`);
+                  for (const e of extra)
+                    diff.push(`+ unexpected function: ${e}`);
+                  const usedActual = /* @__PURE__ */ new Set();
+                  for (const expectedObj of gtArr) {
+                    const fname = Object.keys(expectedObj)[0];
+                    let matchedIndex = -1;
+                    for (let i = 0; i < restoredCalls.length; i++) {
+                      if (usedActual.has(i)) continue;
+                      const rc = restoredCalls[i];
+                      const rcName = rc?.toolName ?? rc?.name;
+                      if (rcName === fname) {
+                        matchedIndex = i;
+                        break;
+                      }
+                    }
+                    if (matchedIndex === -1) continue;
+                    usedActual.add(matchedIndex);
+                    const received = restoredCalls[matchedIndex];
+                    const receivedArgs = summarizeArgs(received?.args);
+                    const expectedParamsAllowed = expectedObj[fname];
+                    const funcDesc = tools.find(
+                      (t) => t.name === fname
+                    );
+                    const requiredParams = funcDesc?.parameters?.required ?? [];
+                    diff.push(`@@ function ${fname}`);
+                    if (expectedParamsAllowed && receivedArgs && typeof receivedArgs === "object" && receivedArgs !== null) {
+                      for (const req of requiredParams) {
+                        if (!(req in receivedArgs)) {
+                          diff.push(`- missing required param: ${req}`);
+                        }
+                      }
+                      for (const k of Object.keys(
+                        receivedArgs
+                      )) {
+                        if (!Object.prototype.hasOwnProperty.call(
+                          expectedParamsAllowed,
+                          k
+                        )) {
+                          diff.push(`+ unexpected param: ${k}`);
+                        }
+                      }
+                      for (const k of Object.keys(
+                        receivedArgs
+                      )) {
+                        if (Object.prototype.hasOwnProperty.call(
+                          expectedParamsAllowed,
+                          k
+                        )) {
+                          const allowed = expectedParamsAllowed[k];
+                          const got = receivedArgs[k];
+                          const includes = Array.isArray(allowed) && allowed.some((v) => {
+                            try {
+                              if (Array.isArray(got)) {
+                                return JSON.stringify(
+                                  got.map((x) => String(x)).sort()
+                                ) === JSON.stringify(
+                                  v.map((x) => String(x)).sort()
+                                );
+                              }
+                            } catch {
+                            }
+                            return String(v).toLowerCase().replace(/\s+/g, "") === String(got).toLowerCase().replace(/\s+/g, "");
+                          });
+                          if (!includes) {
+                            diff.push(`@@ param ${k}`);
+                            diff.push(
+                              `- expected one of: ${JSON.stringify(allowed)}`
+                            );
+                            diff.push(`+ got: ${JSON.stringify(got)}`);
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+                caseLogs.push(
+                  `[DEBUG-FAIL] ${JSON.stringify({
+                    id: testCase.id,
+                    message: checkerResult.error,
+                    error_type: checkerResult.error_type,
+                    expected,
+                    actual,
+                    diff
+                  })}`
+                );
+              } catch {
+                caseLogs.push(
+                  `[DEBUG] ${testCase.id}: failed to build debug diff`
+                );
+              }
+              return { valid: false, logs: caseLogs };
             }
           } catch (e) {
-            logs.push(
+            caseLogs.push(
               `[ERROR] ${testCase.id}: Model generation failed: ${e?.message}`
             );
             if (e?.stack) {
-              logs.push(`[STACK] ${testCase.id}: ${e.stack}`);
+              caseLogs.push(`[STACK] ${testCase.id}: ${e.stack}`);
             }
+            return { valid: false, logs: caseLogs };
           }
-        }
+        };
+        const mapWithConcurrency = async (items, limit2, mapper) => {
+          const results = new Array(items.length);
+          let idx = 0;
+          const workers = new Array(Math.min(limit2, items.length)).fill(0).map(async () => {
+            while (true) {
+              const current = idx++;
+              if (current >= items.length) break;
+              results[current] = await mapper(items[current], current);
+            }
+          });
+          await Promise.all(workers);
+          return results;
+        };
+        const resultsPerCase = await mapWithConcurrency(
+          testCases,
+          concurrency,
+          async (tc) => runSingleCase(tc)
+        );
+        correctCount = resultsPerCase.reduce(
+          (acc, r) => acc + (r.valid ? 1 : 0),
+          0
+        );
+        for (const r of resultsPerCase) logs.push(...r.logs);
         if (testCases.length === 0) {
           return {
             score: 0,
@@ -830,6 +1068,262 @@ var bfclParallelMultipleBenchmark = createBfclBenchmark(
   "BFCL_v3_parallel_multiple.json",
   "BFCL_v3_parallel_multiple_possible_answer.json"
 );
+
+// src/benchmarks/json-generation.ts
+import { generateText as generateText2 } from "ai";
+import Ajv from "ajv";
+import { promises as fs3 } from "fs";
+import path3 from "path";
+function extractFirstJsonBlock(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+  }
+  const fenceMatch = text.match(/```json\s*([\s\S]*?)```/i) || text.match(/```\s*([\s\S]*?)```/i);
+  if (fenceMatch) {
+    const inner = fenceMatch[1].trim();
+    try {
+      return JSON.parse(inner);
+    } catch {
+    }
+  }
+  const startIdxObj = text.indexOf("{");
+  const startIdxArr = text.indexOf("[");
+  const start = [startIdxObj, startIdxArr].filter((i) => i >= 0).sort((a, b) => a - b)[0];
+  if (start === void 0) return void 0;
+  const open = text[start] === "{" ? "{" : "[";
+  const close = open === "{" ? "}" : "]";
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    const ch = text[i];
+    if (ch === open) depth++;
+    else if (ch === close) depth--;
+    if (depth === 0) {
+      const candidate = text.slice(start, i + 1);
+      try {
+        return JSON.parse(candidate);
+      } catch {
+      }
+      break;
+    }
+  }
+  return void 0;
+}
+function subsetMatch(expected, actual) {
+  if (expected === null || typeof expected !== "object") {
+    return expected === actual;
+  }
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return false;
+    for (let i = 0; i < expected.length; i++) {
+      if (!subsetMatch(expected[i], actual[i])) return false;
+    }
+    return true;
+  }
+  if (actual === null || typeof actual !== "object") return false;
+  const eObj = expected;
+  const aObj = actual;
+  for (const key of Object.keys(eObj)) {
+    if (!subsetMatch(eObj[key], aObj[key])) return false;
+  }
+  return true;
+}
+var jsonGenerationBenchmark = {
+  name: "json-generation",
+  version: "2.1.0",
+  description: "Evaluates schema-compliant JSON generation from natural language using JSON Schema prompts.",
+  async run(model) {
+    const logs = [];
+    const ajv = new Ajv({ allErrors: true, strict: false });
+    let schemaValidCount = 0;
+    let valueMatchCount = 0;
+    let correctCount = 0;
+    let tests = [];
+    const expectedMap = /* @__PURE__ */ new Map();
+    try {
+      const dataDir = resolveDataDir();
+      const testsJsonl = await fs3.readFile(
+        path3.join(dataDir, "json_generation_tests.jsonl"),
+        "utf-8"
+      );
+      const expectedJsonl = await fs3.readFile(
+        path3.join(dataDir, "json_generation_expected.jsonl"),
+        "utf-8"
+      );
+      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+      const expecteds = expectedJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+      for (const r of expecteds) expectedMap.set(r.id, r);
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return {
+        score: 0,
+        success: false,
+        metrics: {},
+        logs: [`[FATAL] Failed to load json-generation datasets: ${msg}`],
+        error: e
+      };
+    }
+    for (const tc of tests) {
+      try {
+        const schemaStr = JSON.stringify(tc.schema, null, 2);
+        const messages = [
+          {
+            role: "system",
+            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
+          },
+          {
+            role: "user",
+            content: [
+              "Generate a JSON object that reflects the following facts.",
+              "JSON Schema:",
+              schemaStr,
+              "Facts:",
+              tc.promptFacts,
+              "Output must be a single JSON only, with no additional text."
+            ].join("\n\n")
+          }
+        ];
+        const { text } = await generateText2({ model, messages });
+        let parsed;
+        try {
+          parsed = extractFirstJsonBlock(text);
+        } catch {
+        }
+        if (parsed === void 0) {
+          logs.push(`[FAIL] ${tc.id}: Unable to parse JSON from model output.`);
+          continue;
+        }
+        const validate = ajv.compile(tc.schema);
+        const valid = validate(parsed);
+        if (valid) schemaValidCount++;
+        else
+          logs.push(
+            `[INFO] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
+          );
+        const expectedRec = expectedMap.get(tc.id);
+        if (!expectedRec) {
+          logs.push(
+            `[WARN] ${tc.id}: No expected record found. Skipping value match.`
+          );
+        }
+        const valuesOk = expectedRec ? subsetMatch(expectedRec.expected, parsed) : false;
+        if (valuesOk) valueMatchCount++;
+        if (valid && valuesOk) {
+          correctCount++;
+          logs.push(`[PASS] ${tc.id}`);
+        } else {
+          logs.push(
+            `[FAIL] ${tc.id}: schemaValid=${valid}, valuesOk=${valuesOk}. Output=${JSON.stringify(
+              parsed
+            )}`
+          );
+        }
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        logs.push(`[ERROR] ${tc.id}: ${msg}`);
+      }
+    }
+    const total = tests.length;
+    const score = correctCount / total;
+    return {
+      score,
+      success: score >= 0.8,
+      metrics: {
+        total_cases: total,
+        correct_count: correctCount,
+        schema_valid_count: schemaValidCount,
+        value_match_count: valueMatchCount,
+        accuracy: score
+      },
+      logs
+    };
+  }
+};
+var jsonGenerationSchemaOnlyBenchmark = {
+  name: "json-generation-schema-only",
+  version: "1.0.1",
+  description: "Evaluates whether model outputs strictly conform to the provided JSON Schema (structure only).",
+  async run(model) {
+    const logs = [];
+    const ajv = new Ajv({ allErrors: true, strict: false });
+    let tests = [];
+    try {
+      const dataDir = resolveDataDir();
+      const testsJsonl = await fs3.readFile(
+        path3.join(dataDir, "json_generation_tests.jsonl"),
+        "utf-8"
+      );
+      tests = testsJsonl.split(/\r?\n/).filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return {
+        score: 0,
+        success: false,
+        metrics: {},
+        logs: [`[FATAL] Failed to load schema-only tests: ${msg}`],
+        error: e
+      };
+    }
+    let schemaValidCount = 0;
+    for (const tc of tests) {
+      try {
+        const schemaStr = JSON.stringify(tc.schema, null, 2);
+        const messages = [
+          {
+            role: "system",
+            content: "You must output only a single JSON document that strictly conforms to the given JSON Schema. Do not include any extra text or code fences."
+          },
+          {
+            role: "user",
+            content: [
+              "Generate a JSON object that reflects the following facts.",
+              "JSON Schema:",
+              schemaStr,
+              "Facts:",
+              tc.promptFacts,
+              "Output must be a single JSON only, with no additional text."
+            ].join("\n\n")
+          }
+        ];
+        const { text } = await generateText2({ model, messages });
+        let parsed;
+        try {
+          parsed = extractFirstJsonBlock(text);
+        } catch {
+        }
+        if (parsed === void 0) {
+          logs.push(`[FAIL] ${tc.id}: Could not parse JSON from model output.`);
+          continue;
+        }
+        const validate = ajv.compile(tc.schema);
+        const valid = validate(parsed);
+        if (valid) {
+          schemaValidCount++;
+          logs.push(`[PASS] ${tc.id}`);
+        } else {
+          logs.push(
+            `[FAIL] ${tc.id}: Schema validation errors: ${(validate.errors || []).map((e) => `${e.instancePath} ${e.message}`).join(", ") || "unknown"}`
+          );
+        }
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        logs.push(`[ERROR] ${tc.id}: ${msg}`);
+      }
+    }
+    const total = tests.length;
+    const score = total > 0 ? schemaValidCount / total : 0;
+    return {
+      score,
+      success: score >= 0.8,
+      metrics: {
+        total_cases: total,
+        schema_valid_count: schemaValidCount,
+        accuracy: score
+      },
+      logs
+    };
+  }
+};
 export {
   bfclMultipleBenchmark,
   bfclParallelBenchmark,