@ai-sdk-tool/eval 1.0.0-canary.0 → 1.0.0-canary.1

package/dist/index.d.cts

@@ -3,7 +3,7 @@ import { LanguageModel } from 'ai';
 /**
  * The result of a single benchmark run.
  */
-type BenchmarkResult = {
+interface BenchmarkResult {
     /**
      * A numeric score for the benchmark.
      * The interpretation of this score is up to the benchmark author.
@@ -27,11 +27,11 @@ type BenchmarkResult = {
      * An error object if the benchmark failed unexpectedly.
      */
     error?: Error;
-};
+}
 /**
  * The interface for defining a language model benchmark.
  */
-type LanguageModelV3Benchmark = {
+interface LanguageModelV3Benchmark {
     /**
      * A unique name for the benchmark.
      */
@@ -51,7 +51,7 @@ type LanguageModelV3Benchmark = {
      * @returns A promise that resolves to a BenchmarkResult.
      */
     run(model: LanguageModel, config?: Record<string, unknown>): Promise<BenchmarkResult>;
-};
+}
 /**
  * The supported reporter types.
  */
@@ -60,17 +60,17 @@ type ReporterType = "console" | "json" | "console.debug";
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
  */
-type EvaluationResult = {
+interface EvaluationResult {
     model: string;
     /** Optional user-provided key when models are passed as a keyed map */
     modelKey?: string;
     benchmark: string;
     result: BenchmarkResult;
-};
+}
 /**
  * Options for the `evaluate` function.
  */
-type EvaluateOptions = {
+interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
      */
@@ -92,16 +92,36 @@ type EvaluateOptions = {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
-};
+}
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelBenchmark: LanguageModelV3Benchmark;
 declare const bfclMultipleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelMultipleBenchmark: LanguageModelV3Benchmark;
 
+/**
+ * ComplexFuncBench - Complex Function Calling Benchmark
+ *
+ * This benchmark evaluates models on complex function calling scenarios including:
+ * - Multi-step function calls in a single turn
+ * - Function calling with constraints
+ * - Parameter value reasoning from implicit information
+ * - Long parameter values (500+ tokens)
+ * - Parallel function calls
+ *
+ * Dataset: https://huggingface.co/datasets/THUDM/ComplexFuncBench
+ * Paper: https://arxiv.org/abs/2501.10132
+ */
+
+/**
+ * ComplexFuncBench benchmark - tests complex function calling scenarios
+ * including multi-step calls, constraints, parameter reasoning, and long parameters.
+ */
+declare const complexFuncBenchBenchmark: LanguageModelV3Benchmark;
+
 declare const jsonGenerationBenchmark: LanguageModelV3Benchmark;
 declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };

package/dist/index.d.ts

@@ -3,7 +3,7 @@ import { LanguageModel } from 'ai';
 /**
  * The result of a single benchmark run.
  */
-type BenchmarkResult = {
+interface BenchmarkResult {
     /**
      * A numeric score for the benchmark.
      * The interpretation of this score is up to the benchmark author.
@@ -27,11 +27,11 @@ type BenchmarkResult = {
      * An error object if the benchmark failed unexpectedly.
      */
     error?: Error;
-};
+}
 /**
  * The interface for defining a language model benchmark.
  */
-type LanguageModelV3Benchmark = {
+interface LanguageModelV3Benchmark {
     /**
      * A unique name for the benchmark.
      */
@@ -51,7 +51,7 @@ type LanguageModelV3Benchmark = {
      * @returns A promise that resolves to a BenchmarkResult.
      */
     run(model: LanguageModel, config?: Record<string, unknown>): Promise<BenchmarkResult>;
-};
+}
 /**
  * The supported reporter types.
  */
@@ -60,17 +60,17 @@ type ReporterType = "console" | "json" | "console.debug";
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
  */
-type EvaluationResult = {
+interface EvaluationResult {
     model: string;
     /** Optional user-provided key when models are passed as a keyed map */
     modelKey?: string;
     benchmark: string;
     result: BenchmarkResult;
-};
+}
 /**
  * Options for the `evaluate` function.
  */
-type EvaluateOptions = {
+interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
      */
@@ -92,16 +92,36 @@ type EvaluateOptions = {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
-};
+}
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelBenchmark: LanguageModelV3Benchmark;
 declare const bfclMultipleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelMultipleBenchmark: LanguageModelV3Benchmark;
 
+/**
+ * ComplexFuncBench - Complex Function Calling Benchmark
+ *
+ * This benchmark evaluates models on complex function calling scenarios including:
+ * - Multi-step function calls in a single turn
+ * - Function calling with constraints
+ * - Parameter value reasoning from implicit information
+ * - Long parameter values (500+ tokens)
+ * - Parallel function calls
+ *
+ * Dataset: https://huggingface.co/datasets/THUDM/ComplexFuncBench
+ * Paper: https://arxiv.org/abs/2501.10132
+ */
+
+/**
+ * ComplexFuncBench benchmark - tests complex function calling scenarios
+ * including multi-step calls, constraints, parameter reasoning, and long parameters.
+ */
+declare const complexFuncBenchBenchmark: LanguageModelV3Benchmark;
+
 declare const jsonGenerationBenchmark: LanguageModelV3Benchmark;
 declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };