@ai-sdk-tool/eval 1.0.0-canary.1 → 1.0.0

package/dist/index.d.cts

@@ -1,5 +1,22 @@
+import { LanguageModelV3Middleware } from '@ai-sdk/provider';
 import { LanguageModel } from 'ai';
 
+/**
+ * Model configuration for evaluation.
+ * Allows specifying a base model with optional middleware for proper cache ordering.
+ */
+interface ModelConfig {
+    /**
+     * The base language model (before any middleware is applied).
+     */
+    model: LanguageModel;
+    /**
+     * Optional middleware to apply to the model.
+     * When cache is enabled, the cache middleware will be applied BEFORE this middleware,
+     * ensuring that cache keys are generated from the final transformed params.
+     */
+    middleware?: LanguageModelV3Middleware | LanguageModelV3Middleware[];
+}
 /**
  * The result of a single benchmark run.
  */
@@ -55,7 +72,7 @@ interface LanguageModelV3Benchmark {
 /**
  * The supported reporter types.
  */
-type ReporterType = "console" | "json" | "console.debug";
+type ReporterType = "console" | "json" | "console.debug" | "console.summary";
 /**
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
@@ -73,8 +90,16 @@ interface EvaluationResult {
 interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
+     * Can be:
+     * - A single LanguageModel or ModelConfig
+     * - An array of LanguageModel or ModelConfig
+     * - A keyed record of LanguageModel or ModelConfig
+     *
+     * When using ModelConfig with middleware and cache enabled,
+     * the cache middleware is applied innermost (closest to the model),
+     * ensuring cache keys reflect the final transformed params.
      */
-    models: LanguageModel | LanguageModel[] | Record<string, LanguageModel>;
+    models: LanguageModel | ModelConfig | (LanguageModel | ModelConfig)[] | Record<string, LanguageModel | ModelConfig>;
     /**
      * An array of benchmarks to run against the models.
      */
@@ -92,6 +117,27 @@ interface EvaluateOptions {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
+    /**
+     * Options for disk-based response caching.
+     * When enabled, LLM responses are cached to disk to avoid redundant API calls.
+     */
+    cache?: {
+        /**
+         * Whether to enable disk caching.
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Directory to store cache files.
+         * @default '.ai-cache'
+         */
+        cacheDir?: string;
+        /**
+         * Whether to log cache hits/misses for debugging.
+         * @default false
+         */
+        debug?: boolean;
+    };
 }
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
@@ -124,4 +170,4 @@ declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ModelConfig, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };

package/dist/index.d.ts

@@ -1,5 +1,22 @@
+import { LanguageModelV3Middleware } from '@ai-sdk/provider';
 import { LanguageModel } from 'ai';
 
+/**
+ * Model configuration for evaluation.
+ * Allows specifying a base model with optional middleware for proper cache ordering.
+ */
+interface ModelConfig {
+    /**
+     * The base language model (before any middleware is applied).
+     */
+    model: LanguageModel;
+    /**
+     * Optional middleware to apply to the model.
+     * When cache is enabled, the cache middleware will be applied BEFORE this middleware,
+     * ensuring that cache keys are generated from the final transformed params.
+     */
+    middleware?: LanguageModelV3Middleware | LanguageModelV3Middleware[];
+}
 /**
  * The result of a single benchmark run.
  */
@@ -55,7 +72,7 @@ interface LanguageModelV3Benchmark {
 /**
  * The supported reporter types.
  */
-type ReporterType = "console" | "json" | "console.debug";
+type ReporterType = "console" | "json" | "console.debug" | "console.summary";
 /**
  * The full result object for an evaluation run,
  * containing results for all model-benchmark combinations.
@@ -73,8 +90,16 @@ interface EvaluationResult {
 interface EvaluateOptions {
     /**
      * The language model or models to evaluate.
+     * Can be:
+     * - A single LanguageModel or ModelConfig
+     * - An array of LanguageModel or ModelConfig
+     * - A keyed record of LanguageModel or ModelConfig
+     *
+     * When using ModelConfig with middleware and cache enabled,
+     * the cache middleware is applied innermost (closest to the model),
+     * ensuring cache keys reflect the final transformed params.
      */
-    models: LanguageModel | LanguageModel[] | Record<string, LanguageModel>;
+    models: LanguageModel | ModelConfig | (LanguageModel | ModelConfig)[] | Record<string, LanguageModel | ModelConfig>;
     /**
      * An array of benchmarks to run against the models.
      */
@@ -92,6 +117,27 @@ interface EvaluateOptions {
      * Optional maximum number of tokens to generate during evaluation.
      */
     maxTokens?: number;
+    /**
+     * Options for disk-based response caching.
+     * When enabled, LLM responses are cached to disk to avoid redundant API calls.
+     */
+    cache?: {
+        /**
+         * Whether to enable disk caching.
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Directory to store cache files.
+         * @default '.ai-cache'
+         */
+        cacheDir?: string;
+        /**
+         * Whether to log cache hits/misses for debugging.
+         * @default false
+         */
+        debug?: boolean;
+    };
 }
 
 declare const bfclSimpleBenchmark: LanguageModelV3Benchmark;
@@ -124,4 +170,4 @@ declare const jsonGenerationSchemaOnlyBenchmark: LanguageModelV3Benchmark;
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;
 
-export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };
+export { type BenchmarkResult, type EvaluateOptions, type LanguageModelV3Benchmark, type ModelConfig, type ReporterType, bfclMultipleBenchmark, bfclParallelBenchmark, bfclParallelMultipleBenchmark, bfclSimpleBenchmark, complexFuncBenchBenchmark, evaluate, jsonGenerationBenchmark, jsonGenerationSchemaOnlyBenchmark };