@m4trix/evals 0.14.0 → 0.16.0

package/dist/cli.js

@@ -11,7 +11,7 @@ import { existsSync } from 'fs';
 import * as jitiModule from 'jiti';
 import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffLines } from 'diff';
+import { diffString } from 'json-diff';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -978,45 +978,46 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
-function toJsonLines(value) {
+function createDiffString(expected, actual, diffOptions) {
+  const opts = { ...diffOptions, color: false };
+  const result = diffString(expected, actual, opts);
+  return typeof result === "string" ? result : "";
+}
+function formatLogMessage(msg) {
+  if (typeof msg === "string")
+    return msg;
   try {
-    return JSON.stringify(value, null, 2);
+    if (msg !== null && typeof msg === "object") {
+      return JSON.stringify(msg, null, 2);
+    }
+    return String(msg);
   } catch {
-    return String(value);
+    return String(msg);
   }
 }
-function formatDiffString(changes) {
-  const lines = [];
-  for (const part of changes) {
-    const prefix = part.added ? "+" : part.removed ? "-" : " ";
-    const partLines = part.value.split("\n");
-    if (partLines[partLines.length - 1] === "") {
-      partLines.pop();
-    }
-    for (const line of partLines) {
-      lines.push(`${prefix} ${line}`);
-    }
-  }
-  return lines.join("\n");
+function createLogEntry(message, options) {
+  return {
+    type: "log",
+    label: options?.label,
+    message: formatLogMessage(message)
+  };
 }
-function createDiffString(expected, actual) {
-  const expectedStr = toJsonLines(expected);
-  const actualStr = toJsonLines(actual);
-  const changes = diffLines(expectedStr, actualStr);
-  return formatDiffString(changes);
+function getLogLines(entry) {
+  return entry.message.split("\n");
 }
 function createDiffLogEntry(expected, actual, options) {
-  const diff = createDiffString(expected, actual);
+  const { label, ...diffOpts } = options ?? {};
+  const diff = createDiffString(expected, actual, diffOpts);
   return {
     type: "diff",
-    label: options?.label,
+    label,
     expected,
     actual,
     diff: diff || "(no differences)"
   };
 }
 function getDiffLines(entry) {
-  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
+  const raw = entry.diff || "(no differences)";
   return raw.split("\n").map((line) => {
     const trimmed = line.trimStart();
     if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -1274,6 +1275,9 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
           const logDiff = (expected, actual, options) => {
             logs.push(createDiffLogEntry(expected, actual, options));
           };
+          const log = (message, options) => {
+            logs.push(createLogEntry(message, options));
+          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
@@ -1283,7 +1287,8 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
-                logDiff
+                logDiff,
+                log
               })
             )
           );
@@ -2285,26 +2290,60 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
             ":",
             " ",
             /* @__PURE__ */ jsx(Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-            " ",
-            item.scores.map((s) => /* @__PURE__ */ jsxs(Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
-              formatScorePart(s),
-              " "
-            ] }, s.id)),
-            item.metrics?.map((m) => {
-              const def = getMetricById(m.id);
-              if (!def)
-                return null;
-              const formatted = def.format(m.data);
-              return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-                "[",
-                def.name ? `${def.name}: ` : "",
-                formatted,
-                "]",
-                " "
-              ] }, m.id);
-            })
+            item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxs(Fragment, { children: [
+              " ",
+              item.metrics.map((m) => {
+                const def = getMetricById(m.id);
+                if (!def)
+                  return null;
+                const formatted = def.format(m.data);
+                return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+                  "[",
+                  def.name ? `${def.name}: ` : "",
+                  formatted,
+                  "]",
+                  " "
+                ] }, m.id);
+              })
+            ] }) : null
           ] }, `tc-${tc.testCaseId}-${item.evaluatorId}`)
         );
+        if (item.scores.length > 0) {
+          for (let sIdx = 0; sIdx < item.scores.length; sIdx++) {
+            const s = item.scores[sIdx];
+            const def = getScoreById(s.id);
+            const scoreLabel = def ? def.name ?? def.id : s.id;
+            rows.push(
+              /* @__PURE__ */ jsxs(
+                Text,
+                {
+                  color: scoreColor(toNumericScore(s.data) ?? 0),
+                  children: [
+                    "      ",
+                    scoreLabel,
+                    ": ",
+                    formatScorePart(s)
+                  ]
+                },
+                `tc-${tc.testCaseId}-${item.evaluatorId}-score-${sIdx}`
+              )
+            );
+          }
+        } else {
+          rows.push(
+            /* @__PURE__ */ jsxs(
+              Text,
+              {
+                color: "gray",
+                children: [
+                  "      ",
+                  "n/a"
+                ]
+              },
+              `tc-${tc.testCaseId}-${item.evaluatorId}-n/a`
+            )
+          );
+        }
         if (!item.passed && item.logs && item.logs.length > 0) {
           for (let logIdx = 0; logIdx < item.logs.length; logIdx++) {
             const log = item.logs[logIdx];
@@ -2326,6 +2365,23 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
                   )
                 );
               }
+            } else if (log.type === "log") {
+              const logLines = getLogLines(log);
+              for (let lineIdx = 0; lineIdx < logLines.length; lineIdx++) {
+                rows.push(
+                  /* @__PURE__ */ jsxs(
+                    Text,
+                    {
+                      color: "gray",
+                      children: [
+                        "      ",
+                        logLines[lineIdx]
+                      ]
+                    },
+                    `tc-${tc.testCaseId}-${item.evaluatorId}-${logIdx}-${lineIdx}`
+                  )
+                );
+              }
             }
           }
         }