@ai-sdk-tool/eval 0.1.4 → 0.1.6

package/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2025 Woonggi Min (https://github.com/minpeter)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/README.md

@@ -122,3 +122,7 @@ export const politenessBenchmark: LanguageModelV2Benchmark = {
 //   benchmarks: [politenessBenchmark],
 // });
 ```
+
+## License
+
+Licensed under Apache License 2.0. See the repository `LICENSE`. Include the `NOTICE` file in distributions.

package/dist/index.cjs

@@ -281,13 +281,13 @@ var reporters = {
 };
 
 // src/evaluate.ts
-async function runSingleBenchmark(model, benchmark, modelKey) {
+async function runSingleBenchmark(model, benchmark, modelKey, config) {
   const modelId = typeof model === "object" && model !== null && "modelId" in model && typeof model.modelId === "string" ? model.modelId : "unknown-model";
   try {
     console.log(
       `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Running benchmark: ${benchmark.name}...`
     );
-    const result = await benchmark.run(model);
+    const result = await benchmark.run(model, config);
     console.log(
       `[${modelId}]${modelKey ? ` (${modelKey})` : ""} Finished benchmark: ${benchmark.name}. Score: ${result.score}`
     );
@@ -316,7 +316,7 @@ async function runSingleBenchmark(model, benchmark, modelKey) {
   }
 }
 async function evaluate(options) {
-  const { models, benchmarks, reporter = "console" } = options;
+  const { models, benchmarks, reporter = "console", temperature } = options;
   const modelEntries = [];
   if (Array.isArray(models)) {
     for (const m of models) modelEntries.push([void 0, m]);
@@ -335,7 +335,8 @@ async function evaluate(options) {
       const evaluationResult = await runSingleBenchmark(
         model,
         benchmark,
-        modelKey
+        modelKey,
+        temperature !== void 0 ? { temperature } : void 0
       );
       allResults.push(evaluationResult);
     }
@@ -675,7 +676,7 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
     name,
     version: "1.0.0",
     description,
-    async run(model) {
+    async run(model, config) {
       const logs = [];
       let correctCount = 0;
       let testCases = [];
@@ -733,6 +734,8 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
         const runSingleCase = async (testCase) => {
           const caseLogs = [];
           const { function: tools, question: messages } = testCase;
+          const temp = config?.temperature;
+          const temperature = typeof temp === "number" ? temp : void 0;
           try {
             const flatMessages = Array.isArray(messages) && messages.some((m) => Array.isArray(m)) ? messages.flat(1) : messages;
             const nameMap = /* @__PURE__ */ new Map();
@@ -778,6 +781,7 @@ function createBfclBenchmark(name, description, testDataFile, answerDataFile) {
               messages: flatMessages,
               tools: toolsMap,
               toolChoice: "auto",
+              ...temperature !== void 0 ? { temperature } : {},
               // Pass original schema information to middleware
               providerOptions: {
                 toolCallMiddleware: {
@@ -1174,7 +1178,7 @@ var jsonGenerationBenchmark = {
   name: "json-generation",
   version: "2.1.0",
   description: "Evaluates schema-compliant JSON generation from natural language using JSON Schema prompts.",
-  async run(model) {
+  async run(model, config) {
     const logs = [];
     const ajv = new import_ajv.default({ allErrors: true, strict: false });
     let schemaValidCount = 0;
@@ -1225,7 +1229,13 @@ var jsonGenerationBenchmark = {
             ].join("\n\n")
           }
         ];
-        const { text } = await (0, import_ai2.generateText)({ model, messages });
+        const temp = config?.temperature;
+        const temperature = typeof temp === "number" ? temp : void 0;
+        const { text } = await (0, import_ai2.generateText)({
+          model,
+          messages,
+          ...temperature !== void 0 ? { temperature } : {}
+        });
         let parsed;
         try {
           parsed = extractFirstJsonBlock(text);
@@ -1285,7 +1295,7 @@ var jsonGenerationSchemaOnlyBenchmark = {
   name: "json-generation-schema-only",
   version: "1.0.1",
   description: "Evaluates whether model outputs strictly conform to the provided JSON Schema (structure only).",
-  async run(model) {
+  async run(model, config) {
     const logs = [];
     const ajv = new import_ajv.default({ allErrors: true, strict: false });
     let tests = [];
@@ -1327,7 +1337,13 @@ var jsonGenerationSchemaOnlyBenchmark = {
             ].join("\n\n")
           }
         ];
-        const { text } = await (0, import_ai2.generateText)({ model, messages });
+        const temp = config?.temperature;
+        const temperature = typeof temp === "number" ? temp : void 0;
+        const { text } = await (0, import_ai2.generateText)({
+          model,
+          messages,
+          ...temperature !== void 0 ? { temperature } : {}
+        });
         let parsed;
         try {
           parsed = extractFirstJsonBlock(text);