@ai-sdk-tool/eval 1.0.0-canary.0 → 1.0.0

package/dist/index.d.cts

@@ -1,9 +1,26 @@
+import { LanguageModelV3Middleware } from '@ai-sdk/provider';
 import { LanguageModel } from 'ai';
 
+/**
+ * Model configuration for evaluation.
+ * Allows specifying a base model with optional middleware for proper cache ordering.
+ */
+interface ModelConfig {
+    /**
+     * The base language model (before any middleware is applied).
+     */
+    model: LanguageModel;
+    /**
+     * Optional middleware to apply to the model.
+     * When cache is enabled, the cache middleware will be applied BEFORE this middleware,
+     * ensuring that cache keys are generated from the final transformed params.
+     */
+    middleware?: LanguageModelV3Middleware | LanguageModelV3Middleware[];
+}
 /**
  * The result of a single benchmark run.
  */
-type BenchmarkResult = {
+interface BenchmarkResult {
     /**
      * A numeric score for the benchmark.
      * The interpretation of this score is up to the benchmark author.
@@ -27,11 +44,11 @@ type BenchmarkResult = {
      * An error object if the benchmark failed unexpectedly.
      */
     error?: Error;
-};
+}
 /**
  * The interface for defining a language model benchmark.
  */
-type LanguageModelV3Benchmark = {
+interface LanguageModelV3Benchmark {
     /**
      * A unique name for the benchmark.
      */
@@ -51,30 +68,38 @@ type LanguageModelV3Benchmark = {
      * @returns A promise that resolves to a BenchmarkResult.
      */
     run(model: LanguageModel, config?: Record<string, unknown>): Promise<BenchmarkResult>;
-};
+}
 /**
  * The supported reporter types.
  */
-type ReporterType = "console" | "json" | "console.debug";
+type ReporterType = "console" | "json" | "console.debug" | "console.summary";
 /**
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
  */
-type EvaluationResult = {
+interface EvaluationResult {
     model: string;
     /** Optional user-provided key when models are passed as a keyed map */
     modelKey?: string;
     benchmark: string;
     result: BenchmarkResult;
-};
+}
 /**
  * Options for the `evaluate` function.
  */
-type EvaluateOptions = {
+interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
+     * Can be:
+     * - A single LanguageModel or ModelConfig
+     * - An array of LanguageModel or ModelConfig
+     * - A keyed record of LanguageModel or ModelConfig
+     *
+     * When using ModelConfig with middleware and cache enabled,
+     * the cache middleware is applied innermost (closest to the model),
+     * ensuring cache keys reflect the final transformed params.
      */
-    models: LanguageModel | LanguageModel[] | Record<string, LanguageModel>;
+    models: LanguageModel | ModelConfig | (LanguageModel | ModelConfig)[] | Record<string, LanguageModel | ModelConfig>;
     /**
      * An array of benchmarks to run against the models.
      */
@@ -92,16 +117,57 @@ type EvaluateOptions = {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
-};
+    /**
+     * Options for disk-based response caching.
+     * When enabled, LLM responses are cached to disk to avoid redundant API calls.
+     */
+    cache?: {
+        /**
+         * Whether to enable disk caching.
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Directory to store cache files.
+         * @default '.ai-cache'
+         */
+        cacheDir?: string;
+        /**
+         * Whether to log cache hits/misses for debugging.
+         * @default false
+         */
+        debug?: boolean;
+    };
+}
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelBenchmark: LanguageModelV3Benchmark;
 declare const bfclMultipleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelMultipleBenchmark: LanguageModelV3Benchmark;
 
+/**
+ * ComplexFuncBench - Complex Function Calling Benchmark
+ *
+ * This benchmark evaluates models on complex function calling scenarios including:
+ * - Multi-step function calls in a single turn
+ * - Function calling with constraints
+ * - Parameter value reasoning from implicit information
+ * - Long parameter values (500+ tokens)
+ * - Parallel function calls
+ *
+ * Dataset: https://huggingface.co/datasets/THUDM/ComplexFuncBench
+ * Paper: https://arxiv.org/abs/2501.10132
+ */
+
+/**
+ * ComplexFuncBench benchmark - tests complex function calling scenarios
+ * including multi-step calls, constraints, parameter reasoning, and long parameters.
+ */
+declare const complexFuncBenchBenchmark: LanguageModelV3Benchmark;
+
 declare const jsonGenerationBenchmark: LanguageModelV3Benchmark;
 declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ModelConfig, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };

package/dist/index.d.ts

@@ -1,9 +1,26 @@
+import { LanguageModelV3Middleware } from '@ai-sdk/provider';
 import { LanguageModel } from 'ai';
 
+/**
+ * Model configuration for evaluation.
+ * Allows specifying a base model with optional middleware for proper cache ordering.
+ */
+interface ModelConfig {
+    /**
+     * The base language model (before any middleware is applied).
+     */
+    model: LanguageModel;
+    /**
+     * Optional middleware to apply to the model.
+     * When cache is enabled, the cache middleware will be applied BEFORE this middleware,
+     * ensuring that cache keys are generated from the final transformed params.
+     */
+    middleware?: LanguageModelV3Middleware | LanguageModelV3Middleware[];
+}
 /**
  * The result of a single benchmark run.
  */
-type BenchmarkResult = {
+interface BenchmarkResult {
     /**
      * A numeric score for the benchmark.
      * The interpretation of this score is up to the benchmark author.
@@ -27,11 +44,11 @@ type BenchmarkResult = {
      * An error object if the benchmark failed unexpectedly.
      */
     error?: Error;
-};
+}
 /**
  * The interface for defining a language model benchmark.
  */
-type LanguageModelV3Benchmark = {
+interface LanguageModelV3Benchmark {
     /**
      * A unique name for the benchmark.
      */
@@ -51,30 +68,38 @@ type LanguageModelV3Benchmark = {
      * @returns A promise that resolves to a BenchmarkResult.
      */
     run(model: LanguageModel, config?: Record<string, unknown>): Promise<BenchmarkResult>;
-};
+}
 /**
  * The supported reporter types.
  */
-type ReporterType = "console" | "json" | "console.debug";
+type ReporterType = "console" | "json" | "console.debug" | "console.summary";
 /**
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
  */
-type EvaluationResult = {
+interface EvaluationResult {
     model: string;
     /** Optional user-provided key when models are passed as a keyed map */
     modelKey?: string;
     benchmark: string;
     result: BenchmarkResult;
-};
+}
 /**
  * Options for the `evaluate` function.
  */
-type EvaluateOptions = {
+interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
+     * Can be:
+     * - A single LanguageModel or ModelConfig
+     * - An array of LanguageModel or ModelConfig
+     * - A keyed record of LanguageModel or ModelConfig
+     *
+     * When using ModelConfig with middleware and cache enabled,
+     * the cache middleware is applied innermost (closest to the model),
+     * ensuring cache keys reflect the final transformed params.
      */
-    models: LanguageModel | LanguageModel[] | Record<string, LanguageModel>;
+    models: LanguageModel | ModelConfig | (LanguageModel | ModelConfig)[] | Record<string, LanguageModel | ModelConfig>;
     /**
      * An array of benchmarks to run against the models.
      */
@@ -92,16 +117,57 @@ type EvaluateOptions = {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
-};
+    /**
+     * Options for disk-based response caching.
+     * When enabled, LLM responses are cached to disk to avoid redundant API calls.
+     */
+    cache?: {
+        /**
+         * Whether to enable disk caching.
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Directory to store cache files.
+         * @default '.ai-cache'
+         */
+        cacheDir?: string;
+        /**
+         * Whether to log cache hits/misses for debugging.
+         * @default false
+         */
+        debug?: boolean;
+    };
+}
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelBenchmark: LanguageModelV3Benchmark;
 declare const bfclMultipleBenchmark: LanguageModelV3Benchmark;
 declare const bfclParallelMultipleBenchmark: LanguageModelV3Benchmark;
 
+/**
+ * ComplexFuncBench - Complex Function Calling Benchmark
+ *
+ * This benchmark evaluates models on complex function calling scenarios including:
+ * - Multi-step function calls in a single turn
+ * - Function calling with constraints
+ * - Parameter value reasoning from implicit information
+ * - Long parameter values (500+ tokens)
+ * - Parallel function calls
+ *
+ * Dataset: https://huggingface.co/datasets/THUDM/ComplexFuncBench
+ * Paper: https://arxiv.org/abs/2501.10132
+ */
+
+/**
+ * ComplexFuncBench benchmark - tests complex function calling scenarios
+ * including multi-step calls, constraints, parameter reasoning, and long parameters.
+ */
+declare const complexFuncBenchBenchmark: LanguageModelV3Benchmark;
+
 declare const jsonGenerationBenchmark: LanguageModelV3Benchmark;
 declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ModelConfig, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };