@m4trix/evals 0.14.0 → 0.16.0

package/dist/cli-simple.js

@@ -6,7 +6,7 @@ import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffLines } from 'diff';
+import { diffString } from 'json-diff';
 import React2, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
@@ -260,45 +260,46 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
-function toJsonLines(value) {
+function createDiffString(expected, actual, diffOptions) {
+  const opts = { ...diffOptions, color: false };
+  const result = diffString(expected, actual, opts);
+  return typeof result === "string" ? result : "";
+}
+function formatLogMessage(msg) {
+  if (typeof msg === "string")
+    return msg;
   try {
-    return JSON.stringify(value, null, 2);
+    if (msg !== null && typeof msg === "object") {
+      return JSON.stringify(msg, null, 2);
+    }
+    return String(msg);
   } catch {
-    return String(value);
+    return String(msg);
   }
 }
-function formatDiffString(changes) {
-  const lines = [];
-  for (const part of changes) {
-    const prefix = part.added ? "+" : part.removed ? "-" : " ";
-    const partLines = part.value.split("\n");
-    if (partLines[partLines.length - 1] === "") {
-      partLines.pop();
-    }
-    for (const line of partLines) {
-      lines.push(`${prefix} ${line}`);
-    }
-  }
-  return lines.join("\n");
+function createLogEntry(message, options) {
+  return {
+    type: "log",
+    label: options?.label,
+    message: formatLogMessage(message)
+  };
 }
-function createDiffString(expected, actual) {
-  const expectedStr = toJsonLines(expected);
-  const actualStr = toJsonLines(actual);
-  const changes = diffLines(expectedStr, actualStr);
-  return formatDiffString(changes);
+function getLogLines(entry) {
+  return entry.message.split("\n");
 }
 function createDiffLogEntry(expected, actual, options) {
-  const diff = createDiffString(expected, actual);
+  const { label, ...diffOpts } = options ?? {};
+  const diff = createDiffString(expected, actual, diffOpts);
   return {
     type: "diff",
-    label: options?.label,
+    label,
     expected,
     actual,
     diff: diff || "(no differences)"
   };
 }
 function getDiffLines(entry) {
-  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
+  const raw = entry.diff || "(no differences)";
   return raw.split("\n").map((line) => {
     const trimmed = line.trimStart();
     if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -574,6 +575,9 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
           const logDiff = (expected, actual, options) => {
             logs.push(createDiffLogEntry(expected, actual, options));
           };
+          const log = (message, options) => {
+            logs.push(createLogEntry(message, options));
+          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
@@ -583,7 +587,8 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
-                logDiff
+                logDiff,
+                log
               })
             )
           );
@@ -1518,6 +1523,7 @@ function RunView({
     );
     setEvaluatorNameById(nameById);
     const aggregates = /* @__PURE__ */ new Map();
+    const scoreItemsByEvaluatorScore = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
     let overallScoreSumSq = 0;
     let overallScoreCount = 0;
@@ -1547,6 +1553,12 @@ function RunView({
               overallScoreSumSq += numeric * numeric;
               overallScoreCount += 1;
             }
+            for (const s of item.scores) {
+              const key = `${item.evaluatorId}:${s.id}`;
+              const list = scoreItemsByEvaluatorScore.get(key) ?? [];
+              list.push(s);
+              scoreItemsByEvaluatorScore.set(key, list);
+            }
           }
           setTestCases((prev) => {
             const byId = new Map(prev.map((tc) => [tc.testCaseId, tc]));
@@ -1617,6 +1629,7 @@ function RunView({
       overallScoreSumSq,
       overallScoreCount,
       aggregates: new Map(aggregates),
+      scoreItemsByEvaluatorScore: new Map(scoreItemsByEvaluatorScore),
       artifactPath: finalEvent.artifactPath
     });
     setPhase("completed");
@@ -1699,36 +1712,45 @@ function RunView({
               ":",
               " ",
               /* @__PURE__ */ jsx(Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-              " ",
-              item.scores.map((s) => /* @__PURE__ */ jsxs(
+              item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxs(Fragment, { children: [
+                " ",
+                item.metrics.map((m) => {
+                  const def = getMetricById(m.id);
+                  if (!def)
+                    return null;
+                  const formatted = def.format(m.data, {
+                    isAggregated: tc.isAggregated
+                  });
+                  return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+                    "[",
+                    def.name ? `${def.name}: ` : "",
+                    formatted,
+                    "]",
+                    " "
+                  ] }, m.id);
+                })
+              ] }) : null
+            ] }),
+            item.scores.length > 0 ? item.scores.map((s, idx) => {
+              const def = getScoreById(s.id);
+              const scoreLabel = def ? def.name ?? def.id : s.id;
+              return /* @__PURE__ */ jsxs(
                 Text,
                 {
                   color: scoreColor(toNumericScore(s.data) ?? 0),
                   children: [
+                    "      ",
+                    scoreLabel,
+                    ":",
+                    " ",
                     formatScorePart(s, scoreColor, {
                       isAggregated: tc.isAggregated
-                    }),
-                    " "
+                    })
                   ]
                 },
-                s.id
-              )),
-              item.metrics?.map((m) => {
-                const def = getMetricById(m.id);
-                if (!def)
-                  return null;
-                const formatted = def.format(m.data, {
-                  isAggregated: tc.isAggregated
-                });
-                return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-                  "[",
-                  def.name ? `${def.name}: ` : "",
-                  formatted,
-                  "]",
-                  " "
-                ] }, m.id);
-              })
-            ] }),
+                `${item.evaluatorId}-${s.id}-${idx}`
+              );
+            }) : /* @__PURE__ */ jsx(Text, { color: "gray", children: "      n/a" }),
             !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsx(Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
               (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getDiffLines(log).map(
                 ({ type, line }, lineIdx) => /* @__PURE__ */ jsx(
@@ -1739,7 +1761,7 @@ function RunView({
                   },
                   lineIdx
                 )
-              ) }, logIdx) : null
+              ) }, logIdx) : log.type === "log" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getLogLines(log).map((line, lineIdx) => /* @__PURE__ */ jsx(Text, { color: "gray", children: line }, lineIdx)) }, logIdx) : null
             ) })
           ]
         },
@@ -1786,26 +1808,54 @@ function RunView({
         /* @__PURE__ */ jsx(Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          if (!agg || agg.count === 0) {
+          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
+            (k) => k.startsWith(`${id}:`)
+          );
+          if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
               "- ",
               name.padEnd(28),
-              " no numeric scores"
+              " no scores"
             ] }, id);
           }
-          const mean = agg.total / agg.count;
-          const sd = sampleStdDev(agg.total, agg.sumSq, agg.count);
-          const meanStr = sd !== void 0 ? `${mean.toFixed(2)} \xB1 ${sd.toFixed(2)}` : mean.toFixed(2);
-          return /* @__PURE__ */ jsxs(Text, { children: [
-            "- ",
-            name.padEnd(28),
-            " avg=",
-            /* @__PURE__ */ jsx(Text, { color: scoreColor(mean), children: meanStr }),
+          const passedFailed = agg != null ? /* @__PURE__ */ jsxs(Text, { children: [
             " ",
             "passed=",
             agg.passed,
             " failed=",
             agg.failed
+          ] }) : null;
+          return /* @__PURE__ */ jsxs(Box, { flexDirection: "column", children: [
+            /* @__PURE__ */ jsxs(Text, { children: [
+              "- ",
+              name.padEnd(28),
+              passedFailed
+            ] }),
+            scoreKeys.map((key) => {
+              const items = summary.scoreItemsByEvaluatorScore?.get(key) ?? [];
+              const aggregated = aggregateScoreItems(items);
+              if (!aggregated)
+                return null;
+              const def = getScoreById(aggregated.id);
+              const label = def ? def.name ?? def.id : aggregated.id;
+              const formatted = def?.format(aggregated.data, {
+                isAggregated: true
+              }) ?? "n/a";
+              const numeric = toNumericScore(aggregated.data);
+              return /* @__PURE__ */ jsxs(
+                Text,
+                {
+                  color: numeric !== void 0 ? scoreColor(numeric) : "gray",
+                  children: [
+                    "    ",
+                    label,
+                    ": ",
+                    formatted
+                  ]
+                },
+                key
+              );
+            })
           ] }, id);
         })
       ] }),
@@ -1933,14 +1983,36 @@ function scoreToColor(score) {
   }
   return ansi2.red;
 }
-function getEvaluatorSummaryLine(evaluatorName, aggregate) {
-  if (!aggregate || aggregate.count === 0) {
-    return `- ${evaluatorName.padEnd(28)} no numeric scores`;
+function getEvaluatorSummaryLines(evaluatorId, evaluatorName, aggregate, scoreItemsByKey) {
+  const lines = [];
+  const scoreKeys = [...scoreItemsByKey.keys()].filter(
+    (k) => k.startsWith(`${evaluatorId}:`)
+  );
+  if (scoreKeys.length === 0) {
+    lines.push(`- ${evaluatorName.padEnd(28)} no scores`);
+    return lines;
+  }
+  const passedFailed = aggregate != null ? ` passed=${aggregate.passed} failed=${aggregate.failed}` : "";
+  const scoreLines = [];
+  for (const key of scoreKeys) {
+    const items = scoreItemsByKey.get(key) ?? [];
+    const agg = aggregateScoreItems(items);
+    if (!agg)
+      continue;
+    const def = getScoreById(agg.id);
+    const label = def ? def.name ?? def.id : agg.id;
+    const formatted = def?.format(agg.data, { isAggregated: true }) ?? "n/a";
+    const numeric = toNumericScore(agg.data);
+    const colored = numeric !== void 0 ? colorize(formatted, scoreToColor(numeric)) : formatted;
+    scoreLines.push(`    ${label}: ${colored}`);
+  }
+  if (scoreLines.length > 0) {
+    lines.push(`- ${evaluatorName.padEnd(28)}${passedFailed}`);
+    lines.push(...scoreLines);
+  } else {
+    lines.push(`- ${evaluatorName.padEnd(28)} no numeric scores${passedFailed}`);
   }
-  const mean = aggregate.total / aggregate.count;
-  const sd = sampleStdDev2(aggregate.total, aggregate.sumSq, aggregate.count);
-  const meanStr = sd !== void 0 ? `${mean.toFixed(2)} \xB1 ${sd.toFixed(2)}` : mean.toFixed(2);
-  return `- ${evaluatorName.padEnd(28)} avg=${colorize(meanStr, scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;
+  return lines;
 }
 function createBar2(value, max = 100, width = 20) {
   const safe = Math.max(0, Math.min(max, value));
@@ -1992,46 +2064,8 @@ function aggregateEvaluatorScoresFromEvents(events, evaluatorNameById) {
 }
 function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   const passLabel = passed ? colorize("PASS", `${ansi2.bold}${ansi2.green}`) : colorize("FAIL", `${ansi2.bold}${ansi2.red}`);
-  const scoreParts = [];
-  for (const item of scores) {
-    const def = getScoreById(item.id);
-    if (!def) {
-      const numeric = toNumericScore(item.data);
-      scoreParts.push(
-        numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a"
-      );
-      continue;
-    }
-    const formatted = def.format(item.data, options);
-    switch (def.displayStrategy) {
-      case "bar": {
-        const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
-        if (typeof numeric === "number" && Number.isFinite(numeric)) {
-          scoreParts.push(
-            `${colorize(formatted, scoreToColor(numeric))} ${colorize(createBar2(numeric), ansi2.dim)}`
-          );
-        } else {
-          scoreParts.push(formatted);
-        }
-        break;
-      }
-      case "number":
-        scoreParts.push(formatted);
-        break;
-      case "passFail":
-        scoreParts.push(
-          colorize(
-            formatted,
-            item.passed === true ? `${ansi2.bold}${ansi2.green}` : item.passed === false ? `${ansi2.bold}${ansi2.red}` : ansi2.dim
-          )
-        );
-        break;
-    }
-  }
-  const scoreStr = scoreParts.length > 0 ? scoreParts.join(" ") : "n/a";
-  let line = `   ${name}: ${passLabel} ${scoreStr}`;
+  const metricParts = [];
   if (metrics && metrics.length > 0) {
-    const metricParts = [];
     for (const { id, data } of metrics) {
       const def = getMetricById(id);
       if (def) {
@@ -2041,11 +2075,49 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
         );
       }
     }
-    if (metricParts.length > 0) {
-      line += ` ${metricParts.join(" ")}`;
+  }
+  const scoreLines = [];
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    const scoreLabel = def ? def.name ?? def.id : item.id;
+    let formatted;
+    if (!def) {
+      const numeric = toNumericScore(item.data);
+      formatted = numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a";
+    } else {
+      const raw = def.format(item.data, options);
+      switch (def.displayStrategy) {
+        case "bar": {
+          const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
+          if (typeof numeric === "number" && Number.isFinite(numeric)) {
+            formatted = `${colorize(raw, scoreToColor(numeric))} ${colorize(createBar2(numeric), ansi2.dim)}`;
+          } else {
+            formatted = raw;
+          }
+          break;
+        }
+        case "number":
+          formatted = raw;
+          break;
+        case "passFail":
+          formatted = colorize(
+            raw,
+            item.passed === true ? `${ansi2.bold}${ansi2.green}` : item.passed === false ? `${ansi2.bold}${ansi2.red}` : ansi2.dim
+          );
+          break;
+      }
     }
+    scoreLines.push(`      ${scoreLabel}: ${formatted}`);
   }
-  return line;
+  const lines = [];
+  const metricStr = metricParts.length > 0 ? ` ${metricParts.join(" ")}` : "";
+  lines.push(`   ${name}: ${passLabel}${metricStr}`);
+  if (scoreLines.length > 0) {
+    lines.push(...scoreLines);
+  } else {
+    lines.push(`      n/a`);
+  }
+  return lines;
 }
 async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern) {
   const dataset = await runner.resolveDatasetByName(datasetName);
@@ -2068,6 +2140,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
   );
   const aggregates = /* @__PURE__ */ new Map();
+  const scoreItemsByEvaluatorScore = /* @__PURE__ */ new Map();
   const testCaseByTestId = /* @__PURE__ */ new Map();
   let overallScoreTotal = 0;
   let overallScoreSumSq = 0;
@@ -2143,6 +2216,12 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
             overallScoreSumSq += numeric * numeric;
             overallScoreCount += 1;
           }
+          for (const s of item.scores) {
+            const key = `${item.evaluatorId}:${s.id}`;
+            const list = scoreItemsByEvaluatorScore.get(key) ?? [];
+            list.push(s);
+            scoreItemsByEvaluatorScore.set(key, list);
+          }
         }
         const isSameTestCase = lastPrintedTestCaseId === testCaseId;
         const isLastRerun = event.rerunIndex >= event.rerunTotal;
@@ -2166,7 +2245,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         for (const item of aggregatedScores) {
           const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
           lines.push(
-            formatEvaluatorScoreLine(
+            ...formatEvaluatorScoreLine(
               name,
               item.scores,
               item.passed,
@@ -2186,6 +2265,10 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
                   const colored = useColor && type === "remove" ? colorize(`      ${line}`, ansi2.red) : useColor && type === "add" ? colorize(`      ${line}`, ansi2.green) : `      ${line}`;
                   lines.push(colored);
                 }
+              } else if (log.type === "log") {
+                for (const line of getLogLines(log)) {
+                  lines.push(`      ${line}`);
+                }
               }
             }
           }
@@ -2263,9 +2346,15 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   }
   console.log(colorize("- evaluator averages:", ansi2.magenta));
   for (const [evaluatorId, evaluatorName] of evaluatorNameById.entries()) {
-    console.log(
-      getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId))
+    const evaluatorLines = getEvaluatorSummaryLines(
+      evaluatorId,
+      evaluatorName,
+      aggregates.get(evaluatorId),
+      scoreItemsByEvaluatorScore
     );
+    for (const line of evaluatorLines) {
+      console.log(line);
+    }
   }
   const testCaseSummaries = buildTestCaseSummaries(testCaseByTestId);
   if (testCaseSummaries.length > 0) {