@m4trix/evals 0.10.0 → 0.11.0

package/dist/cli-simple.js

@@ -6,7 +6,7 @@ import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, mkdir, appendFile, readdir } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import 'json-diff';
+import { diffString } from 'json-diff';
 import React2, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
@@ -256,6 +256,29 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function createDiffLogEntry(expected, actual, options) {
+  const diff = diffString(expected, actual, { color: false });
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function getDiffLines(entry) {
+  const raw = diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
+  return raw.split("\n").map((line) => {
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+      return { type: "remove", line };
+    }
+    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+      return { type: "add", line };
+    }
+    return { type: "context", line };
+  });
+}
 
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
@@ -439,6 +462,10 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
         continue;
       }
       try {
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
         const ctx = yield* Effect.promise(
           () => Promise.resolve(evaluator.resolveContext())
         );
@@ -447,13 +474,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output
+              output,
+              logDiff
             })
           )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
       } catch (error) {
         testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
         evaluatorScores.push({
@@ -1176,7 +1210,8 @@ function RunView({
                 evaluatorName: nameById.get(item.evaluatorId) ?? item.evaluatorId,
                 scores: item.scores,
                 passed: item.passed,
-                metrics: item.metrics
+                metrics: item.metrics,
+                logs: item.logs
               }))
             }
           ]);
@@ -1263,30 +1298,42 @@ function RunView({
           "ms)"
         ] })
       ] }),
-      tc.evaluatorScores.map((item) => /* @__PURE__ */ jsx(Box, { marginLeft: 2, children: /* @__PURE__ */ jsxs(Text, { children: [
-        item.evaluatorName,
-        ":",
-        " ",
-        /* @__PURE__ */ jsx(Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-        " ",
-        item.scores.map((s) => /* @__PURE__ */ jsxs(Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
-          formatScorePart(s),
-          " "
-        ] }, s.id)),
-        item.metrics?.map((m) => {
-          const def = getMetricById(m.id);
-          if (!def)
-            return null;
-          const formatted = def.format(m.data);
-          return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-            "[",
-            def.name ? `${def.name}: ` : "",
-            formatted,
-            "]",
+      tc.evaluatorScores.map((item) => /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginLeft: 2, children: [
+        /* @__PURE__ */ jsxs(Text, { children: [
+          item.evaluatorName,
+          ":",
+          " ",
+          /* @__PURE__ */ jsx(Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
+          " ",
+          item.scores.map((s) => /* @__PURE__ */ jsxs(Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
+            formatScorePart(s),
             " "
-          ] }, m.id);
-        })
-      ] }) }, item.evaluatorId))
+          ] }, s.id)),
+          item.metrics?.map((m) => {
+            const def = getMetricById(m.id);
+            if (!def)
+              return null;
+            const formatted = def.format(m.data);
+            return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+              "[",
+              def.name ? `${def.name}: ` : "",
+              formatted,
+              "]",
+              " "
+            ] }, m.id);
+          })
+        ] }),
+        !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsx(Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
+          (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getDiffLines(log).map(({ type, line }, lineIdx) => /* @__PURE__ */ jsx(
+            Text,
+            {
+              color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+              children: line
+            },
+            lineIdx
+          )) }, logIdx) : null
+        ) })
+      ] }, item.evaluatorId))
     ] }, i)) }),
     phase === "completed" && summary && /* @__PURE__ */ jsxs(Box, { flexDirection: "column", children: [
       /* @__PURE__ */ jsx(Text, { color: "cyan", bold: true, children: "Run Summary" }),
@@ -1535,6 +1582,17 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
               item.metrics
             )
           );
+          if (!item.passed && item.logs && item.logs.length > 0) {
+            for (const log of item.logs) {
+              if (log.type === "diff") {
+                const useColor = process.stdout.isTTY;
+                for (const { type, line } of getDiffLines(log)) {
+                  const colored = useColor && type === "remove" ? colorize(`      ${line}`, ansi2.red) : useColor && type === "add" ? colorize(`      ${line}`, ansi2.green) : `      ${line}`;
+                  console.log(colored);
+                }
+              }
+            }
+          }
           const numeric = toNumericScoreFromScores(item.scores);
           if (numeric !== void 0) {
             const current = aggregates.get(item.evaluatorId) ?? {