@m4trix/evals 0.10.0 → 0.11.0

package/dist/cli.js

@@ -10,7 +10,7 @@ import { resolve, relative, join, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { mkdir, appendFile, readdir } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import 'json-diff';
+import { diffString } from 'json-diff';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -1519,6 +1519,16 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function createDiffLogEntry(expected, actual, options) {
+  const diff = diffString(expected, actual, { color: false });
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
 
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
@@ -1699,6 +1709,10 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
         continue;
       }
       try {
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
         const ctx = yield* Effect.promise(
           () => Promise.resolve(evaluator.resolveContext())
         );
@@ -1707,13 +1721,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output
+              output,
+              logDiff
             })
           )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
       } catch (error) {
         testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
         evaluatorScores.push({