@deepagents/evals 0.20.0 → 0.23.0

package/dist/index.js

@@ -330,8 +330,10 @@ function dataset(source) {
 }
 
 // packages/evals/src/scorers/index.ts
-import { generateObject } from "ai";
-import { z } from "zod";
+import {
+  Factuality as AutoevalsFactuality,
+  Levenshtein as AutoevalsLevenshtein
+} from "autoevals";
 var exactMatch = async ({ output, expected }) => {
   const exp = expected == null ? "" : String(expected);
   if (output === exp) return { score: 1 };
@@ -353,32 +355,32 @@ function regex(pattern) {
     return { score: pattern.test(output) ? 1 : 0 };
   };
 }
-function levenshteinDistance(a, b) {
-  if (a.length === 0) return b.length;
-  if (b.length === 0) return a.length;
-  if (a.length > b.length) [a, b] = [b, a];
-  let prev = Array.from({ length: a.length + 1 }, (_, i) => i);
-  let curr = new Array(a.length + 1);
-  for (let j = 1; j <= b.length; j++) {
-    curr[0] = j;
-    for (let i = 1; i <= a.length; i++) {
-      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
-      curr[i] = Math.min(prev[i] + 1, curr[i - 1] + 1, prev[i - 1] + cost);
+function normalizeScore(score) {
+  if (typeof score !== "number" || !Number.isFinite(score)) return 0;
+  return Math.max(0, Math.min(1, score));
+}
+function reasonFromMetadata(metadata) {
+  if (!metadata) return void 0;
+  const candidates = [
+    metadata.reason,
+    metadata.rationale,
+    metadata.explanation
+  ];
+  for (const candidate of candidates) {
+    if (typeof candidate === "string" && candidate.trim().length > 0) {
+      return candidate;
     }
-    [prev, curr] = [curr, prev];
   }
-  return prev[a.length];
+  return void 0;
 }
 var levenshtein = async ({ output, expected }) => {
   const exp = expected == null ? "" : String(expected);
-  if (output.length === 0 && exp.length === 0) return { score: 1 };
-  const maxLen = Math.max(output.length, exp.length);
-  const distance = levenshteinDistance(output, exp);
-  const score = Math.max(0, 1 - distance / maxLen);
-  if (score === 1) return { score };
+  const result = await AutoevalsLevenshtein({ output, expected: exp });
+  const score = normalizeScore(result.score);
   return {
     score,
-    reason: `Levenshtein distance is ${distance} across max length ${maxLen}.`
+    reason: reasonFromMetadata(result.metadata),
+    metadata: result.metadata
   };
 };
 function deepEqual(a, b) {
@@ -412,42 +414,19 @@ var jsonMatch = async ({ output, expected }) => {
     return { score: 0, reason: "Failed to parse JSON" };
   }
 };
-var llmScorerSchema = z.object({
-  score: z.number().min(0).max(1),
-  reason: z.string()
-});
-function llmJudge(config) {
-  return async ({ input, output, expected }) => {
-    const { object } = await generateObject({
-      model: config.model,
-      schema: llmScorerSchema,
-      prompt: `You are an expert evaluator. Grade the output based on the following criteria:
-${config.criteria}
-
-Input: ${JSON.stringify(input)}
-Output: ${output}
-${expected != null ? `Expected: ${JSON.stringify(expected)}` : ""}
-
-Return a score from 0.0 to 1.0 and a brief reason.`
-    });
-    return { score: object.score, reason: object.reason };
-  };
-}
 function factuality(config) {
   return async ({ input, output, expected }) => {
-    const { object } = await generateObject({
+    const result = await AutoevalsFactuality({
       model: config.model,
-      schema: llmScorerSchema,
-      prompt: `You are a factuality evaluator. Determine whether the output is factually consistent with the expected reference.
-
-Input: ${JSON.stringify(input)}
-Output: ${output}
-Expected reference: ${JSON.stringify(expected)}
-
-Score 1.0 if the output is factually consistent with the reference, 0.0 if it contradicts it. Use intermediate scores for partial consistency.
-Return a score from 0.0 to 1.0 and a brief reason.`
+      input: typeof input === "string" ? input : JSON.stringify(input),
+      output,
+      expected: expected == null ? void 0 : String(expected)
     });
-    return { score: object.score, reason: object.reason };
+    return {
+      score: normalizeScore(result.score),
+      reason: reasonFromMetadata(result.metadata),
+      metadata: result.metadata
+    };
   };
 }
 function all(...scorers) {
@@ -623,6 +602,16 @@ var RunStore = class {
     ).run(id, name, now);
     return { id, name, created_at: now };
   }
+  getSuite(id) {
+    const row = this.#stmt("SELECT * FROM suites WHERE id = ?").get(id);
+    return row ?? void 0;
+  }
+  renameSuite(id, name) {
+    this.#stmt("UPDATE suites SET name = ? WHERE id = ?").run(name, id);
+  }
+  renameRun(id, name) {
+    this.#stmt("UPDATE runs SET name = ? WHERE id = ?").run(name, id);
+  }
   createRun(run) {
     const id = crypto.randomUUID();
     const now = Date.now();
@@ -1042,7 +1031,8 @@ async function runEval(config) {
               });
               scores[sName] = {
                 score: clampScore(sr.score, sName),
-                reason: sr.reason
+                reason: sr.reason,
+                metadata: sr.metadata
               };
             }
             trialResults.push({ result, scores });
@@ -1068,7 +1058,8 @@ async function runEval(config) {
           const meanScore = trialResults.reduce((sum, t) => sum + t.scores[sName].score, 0) / trials;
           finalScores[sName] = {
             score: meanScore,
-            reason: trialResults[trialResults.length - 1].scores[sName]?.reason
+            reason: trialResults[trialResults.length - 1].scores[sName]?.reason,
+            metadata: trialResults[trialResults.length - 1].scores[sName]?.metadata
           };
         }
       } else {
@@ -1085,7 +1076,8 @@ async function runEval(config) {
             });
             finalScores[sName] = {
               score: clampScore(sr.score, sName),
-              reason: sr.reason
+              reason: sr.reason,
+              metadata: sr.metadata
             };
           }
         }
@@ -1455,6 +1447,22 @@ function truncateString(text, maxLength) {
   if (text.length <= maxLength) return text;
   return text.slice(0, maxLength) + "\u2026";
 }
+function stringifyRationale(value) {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : void 0;
+  }
+  if (Array.isArray(value)) {
+    const parts = value.map((item) => typeof item === "string" ? item.trim() : "").filter(Boolean);
+    if (parts.length > 0) return parts.join(" | ");
+  }
+  return void 0;
+}
+function scoreReasonWithMetadata(score) {
+  const reason = score.reason?.trim();
+  if (reason) return reason;
+  return stringifyRationale(score.metadata?.["rationale"]);
+}
 function renderSummaryTable(data) {
   const { summary } = data;
   const passRate = summary.totalCases > 0 ? (summary.passCount / summary.totalCases * 100).toFixed(1) : "0.0";
@@ -1516,7 +1524,8 @@ function renderCaseDetail(c, threshold, options) {
   }
   for (const [name, s] of entries) {
     const scoreColor = s.score >= threshold ? chalk.green : chalk.red;
-    const reasonStr = s.reason ? ` \u2014 ${s.reason}` : "";
+    const reason = scoreReasonWithMetadata(s);
+    const reasonStr = reason ? ` \u2014 ${reason}` : "";
     console.log(
       `    ${chalk.dim(name + ":")} ${scoreColor(s.score.toFixed(3))}${reasonStr}`
     );
@@ -2073,7 +2082,6 @@ export {
   jsonMatch,
   jsonReporter,
   levenshtein,
-  llmJudge,
   markdownReporter,
   parseRecordSelection,
   pickFromArray,