@ai-sdk-tool/eval 0.1.4 → 0.1.5

package/dist/index.d.cts

@@ -84,6 +84,10 @@ interface EvaluateOptions {
      * Defaults to 'console'.
      */
     reporter?: ReporterType;
+    /**
+     * Optional temperature setting to pass to model generation during evaluation.
+     */
+    temperature?: number;
 }
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;

package/dist/index.d.ts

@@ -84,6 +84,10 @@ interface EvaluateOptions {
      * Defaults to 'console'.
      */
     reporter?: ReporterType;
+    /**
+     * Optional temperature setting to pass to model generation during evaluation.
+     */
+    temperature?: number;
 }
 
 declare function evaluate(options: EvaluateOptions): Promise<EvaluationResult[]>;

package/dist/index.js

@@ -239,13 +239,13 @@ var reporters = {
 };
 
 // src/evaluate.ts
-async function runSingleBenchmark(model, benchmark, modelKey) {
+async function runSingleBenchmark(model, benchmark, modelKey, config) {
   const modelId = typeof model === "object" && model !== null && "modelId" in model && typeof model.modelId === "string" ? model.modelId : "unknown-model";
   try {
     console.log(
       `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Running benchmark: ${benchmark.name}...`
     );
-    const result = await benchmark.run(model);
+    const result = await benchmark.run(model, config);
     console.log(
       `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Finished benchmark: ${benchmark.name}. Score: ${result.score}`
     );
@@ -274,7 +274,7 @@ async function runSingleBenchmark(model, benchmark, modelKey) {
   }
 }
 async function evaluate(options) {
-  const { models, benchmarks, reporter = "console" } = options;
+  const { models, benchmarks, reporter = "console", temperature } = options;
   const modelEntries = [];
   if (Array.isArray(models)) {
     for (const m of models) modelEntries.push([void 0, m]);
@@ -293,7 +293,8 @@ async function evaluate(options) {
       const evaluationResult = await runSingleBenchmark(
         model,
         benchmark,
-        modelKey
+        modelKey,
+        temperature !== void 0 ? { temperature } : void 0
       );
       allResults.push(evaluationResult);
     }
@@ -633,7 +634,7 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
     name,
     version: "1.0.0",
     description,
-    async run(model) {
+    async run(model, config) {
       const logs = [];
       let correctCount = 0;
       let testCases = [];
@@ -691,6 +692,8 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
         const runSingleCase = async (testCase) => {
           const caseLogs = [];
           const { function: tools, question: messages } = testCase;
+          const temp = config?.temperature;
+          const temperature = typeof temp === "number" ? temp : void 0;
           try {
             const flatMessages = Array.isArray(messages) && messages.some((m) => Array.isArray(m)) ? messages.flat(1) : messages;
             const nameMap = /* @__PURE__ */ new Map();
@@ -736,6 +739,7 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
               messages: flatMessages,
               tools: toolsMap,
               toolChoice: "auto",
+              ...temperature !== void 0 ? { temperature } : {},
               // Pass original schema information to middleware
               providerOptions: {
                 toolCallMiddleware: {
@@ -1132,7 +1136,7 @@ var jsonGenerationBenchmark = {
   name: "json-generation",
   version: "2.1.0",
   description: "Evaluates schema-compliant JSON generation from natural language using JSON Schema prompts.",
-  async run(model) {
+  async run(model, config) {
     const logs = [];
     const ajv = new Ajv({ allErrors: true, strict: false });
     let schemaValidCount = 0;
@@ -1183,7 +1187,13 @@ var jsonGenerationBenchmark = {
             ].join("\n\n")
           }
         ];
-        const { text } = await generateText2({ model, messages });
+        const temp = config?.temperature;
+        const temperature = typeof temp === "number" ? temp : void 0;
+        const { text } = await generateText2({
+          model,
+          messages,
+          ...temperature !== void 0 ? { temperature } : {}
+        });
         let parsed;
         try {
           parsed = extractFirstJsonBlock(text);
@@ -1243,7 +1253,7 @@ var jsonGenerationSchemaOnlyBenchmark = {
   name: "json-generation-schema-only",
   version: "1.0.1",
   description: "Evaluates whether model outputs strictly conform to the provided JSON Schema (structure only).",
-  async run(model) {
+  async run(model, config) {
     const logs = [];
     const ajv = new Ajv({ allErrors: true, strict: false });
     let tests = [];
@@ -1285,7 +1295,13 @@ var jsonGenerationSchemaOnlyBenchmark = {
             ].join("\n\n")
           }
         ];
-        const { text } = await generateText2({ model, messages });
+        const temp = config?.temperature;
+        const temperature = typeof temp === "number" ? temp : void 0;
+        const { text } = await generateText2({
+          model,
+          messages,
+          ...temperature !== void 0 ? { temperature } : {}
+        });
         let parsed;
         try {
           parsed = extractFirstJsonBlock(text);