@m4trix/evals 0.13.0 → 0.15.0

package/dist/cli-simple.cjs

@@ -8,7 +8,7 @@ var path = require('path');
 var jitiModule = require('jiti');
 var promises = require('fs/promises');
 var url = require('url');
-var jsonDiff = require('json-diff');
+var diff = require('diff');
 var React2 = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
@@ -286,8 +286,35 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diff.diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
 function createDiffLogEntry(expected, actual, options) {
-  const diff = jsonDiff.diffString(expected, actual, { color: false });
+  const diff = createDiffString(expected, actual);
   return {
     type: "diff",
     label: options?.label,
@@ -297,7 +324,7 @@ function createDiffLogEntry(expected, actual, options) {
   };
 }
 function getDiffLines(entry) {
-  const raw = jsonDiff.diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
+  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
   return raw.split("\n").map((line) => {
     const trimmed = line.trimStart();
     if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -357,15 +384,28 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverage(values) {
+function aggregateAverageWithVariance(values) {
   if (values.length === 0) {
-    return { value: 0 };
+    return { value: 0, count: 0 };
   }
   const sum = values.reduce((s, v) => s + v.value, 0);
-  return { value: sum / values.length };
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
 }
 function aggregateAll(values) {
-  return { passed: values.length > 0 && values.every((v) => v.passed) };
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
 }
 function aggregateTokenCountSum(values) {
   const initial = {
@@ -419,14 +459,28 @@ Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
-  aggregate: aggregateAverage
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
 });
 Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
   aggregate: aggregateAll
 });
 
@@ -1365,6 +1419,13 @@ function Spinner({ label = "Running" }) {
     label
   ] });
 }
+function sampleStdDev(sum, sumSq, n) {
+  if (n < 2)
+    return void 0;
+  const mean = sum / n;
+  const variance = (sumSq - n * mean * mean) / (n - 1);
+  return variance > 0 ? Math.sqrt(variance) : 0;
+}
 function scoreColor(score) {
   if (score >= 80)
     return "green";
@@ -1483,7 +1544,9 @@ function RunView({
     );
     setEvaluatorNameById(nameById);
     const aggregates = /* @__PURE__ */ new Map();
+    const scoreItemsByEvaluatorScore = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
+    let overallScoreSumSq = 0;
     let overallScoreCount = 0;
     const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
@@ -1495,19 +1558,28 @@ function RunView({
             if (numeric !== void 0) {
               const current = aggregates.get(item.evaluatorId) ?? {
                 total: 0,
+                sumSq: 0,
                 count: 0,
                 passed: 0,
                 failed: 0
               };
               aggregates.set(item.evaluatorId, {
                 total: current.total + numeric,
+                sumSq: current.sumSq + numeric * numeric,
                 count: current.count + 1,
                 passed: current.passed + (item.passed ? 1 : 0),
                 failed: current.failed + (item.passed ? 0 : 1)
               });
               overallScoreTotal += numeric;
+              overallScoreSumSq += numeric * numeric;
               overallScoreCount += 1;
             }
+            for (const s of item.scores) {
+              const key = `${item.evaluatorId}:${s.id}`;
+              const list = scoreItemsByEvaluatorScore.get(key) ?? [];
+              list.push(s);
+              scoreItemsByEvaluatorScore.set(key, list);
+            }
           }
           setTestCases((prev) => {
             const byId = new Map(prev.map((tc) => [tc.testCaseId, tc]));
@@ -1575,8 +1647,10 @@ function RunView({
       failedTestCases: finalEvent.failedTestCases,
       totalTestCases: finalEvent.totalTestCases,
       overallScoreTotal,
+      overallScoreSumSq,
       overallScoreCount,
       aggregates: new Map(aggregates),
+      scoreItemsByEvaluatorScore: new Map(scoreItemsByEvaluatorScore),
       artifactPath: finalEvent.artifactPath
     });
     setPhase("completed");
@@ -1659,36 +1733,45 @@ function RunView({
               ":",
               " ",
               /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-              " ",
-              item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(
+              item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+                " ",
+                item.metrics.map((m) => {
+                  const def = getMetricById(m.id);
+                  if (!def)
+                    return null;
+                  const formatted = def.format(m.data, {
+                    isAggregated: tc.isAggregated
+                  });
+                  return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+                    "[",
+                    def.name ? `${def.name}: ` : "",
+                    formatted,
+                    "]",
+                    " "
+                  ] }, m.id);
+                })
+              ] }) : null
+            ] }),
+            item.scores.length > 0 ? item.scores.map((s, idx) => {
+              const def = getScoreById(s.id);
+              const scoreLabel = def ? def.name ?? def.id : s.id;
+              return /* @__PURE__ */ jsxRuntime.jsxs(
                 ink.Text,
                 {
                   color: scoreColor(toNumericScore(s.data) ?? 0),
                   children: [
+                    "      ",
+                    scoreLabel,
+                    ":",
+                    " ",
                     formatScorePart(s, scoreColor, {
                       isAggregated: tc.isAggregated
-                    }),
-                    " "
+                    })
                   ]
                 },
-                s.id
-              )),
-              item.metrics?.map((m) => {
-                const def = getMetricById(m.id);
-                if (!def)
-                  return null;
-                const formatted = def.format(m.data, {
-                  isAggregated: tc.isAggregated
-                });
-                return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-                  "[",
-                  def.name ? `${def.name}: ` : "",
-                  formatted,
-                  "]",
-                  " "
-                ] }, m.id);
-              })
-            ] }),
+                `${item.evaluatorId}-${s.id}-${idx}`
+              );
+            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "      n/a" }),
             !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
               (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(
                 ({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
@@ -1732,45 +1815,85 @@ function RunView({
           label: "overall avg",
           value: summary.overallScoreTotal / summary.overallScoreCount,
           barWidth: 20,
-          format: (v) => v.toFixed(2)
+          format: (v) => {
+            const sd = sampleStdDev(
+              summary.overallScoreTotal,
+              summary.overallScoreSumSq,
+              summary.overallScoreCount
+            );
+            return sd !== void 0 ? `${v.toFixed(2)} \xB1 ${sd.toFixed(2)}` : v.toFixed(2);
+          }
         }
       ) }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { marginTop: 1, flexDirection: "column", children: [
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          if (!agg || agg.count === 0) {
+          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
+            (k) => k.startsWith(`${id}:`)
+          );
+          if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
               "- ",
               name.padEnd(28),
-              " no numeric scores"
+              " no scores"
             ] }, id);
           }
-          const mean = agg.total / agg.count;
-          return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-            "- ",
-            name.padEnd(28),
-            " avg=",
-            /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: scoreColor(mean), children: mean.toFixed(2) }),
+          const passedFailed = agg != null ? /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
             " ",
             "passed=",
             agg.passed,
             " failed=",
             agg.failed
+          ] }) : null;
+          return /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
+            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+              "- ",
+              name.padEnd(28),
+              passedFailed
+            ] }),
+            scoreKeys.map((key) => {
+              const items = summary.scoreItemsByEvaluatorScore?.get(key) ?? [];
+              const aggregated = aggregateScoreItems(items);
+              if (!aggregated)
+                return null;
+              const def = getScoreById(aggregated.id);
+              const label = def ? def.name ?? def.id : aggregated.id;
+              const formatted = def?.format(aggregated.data, {
+                isAggregated: true
+              }) ?? "n/a";
+              const numeric = toNumericScore(aggregated.data);
+              return /* @__PURE__ */ jsxRuntime.jsxs(
+                ink.Text,
+                {
+                  color: numeric !== void 0 ? scoreColor(numeric) : "gray",
+                  children: [
+                    "    ",
+                    label,
+                    ": ",
+                    formatted
+                  ]
+                },
+                key
+              );
+            })
           ] }, id);
         })
       ] }),
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { marginTop: 1, flexDirection: "column", children: [
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "test case scores" }),
         testCases.map((tc) => {
-          const numericScores = tc.aggregatedEvaluatorScores.flatMap(
-            (item) => item.scores.map((s) => toNumericScoreFromScores([s])).filter((n) => n !== void 0)
+          const allScores = tc.events.flatMap(
+            (ev) => ev.evaluatorScores.map((es) => toNumericScoreFromScores(es.scores)).filter((n) => n !== void 0)
           );
-          const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
+          const averageScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : void 0;
+          const sumSq = allScores.length > 0 ? allScores.reduce((s, v) => s + v * v, 0) : 0;
+          const total = allScores.reduce((a, b) => a + b, 0);
+          const tcStdDev = sampleStdDev(total, sumSq, allScores.length);
           const firstScore = tc.aggregatedEvaluatorScores[0]?.scores[0];
           const scoreLabel = firstScore && tc.isAggregated ? formatScorePart(firstScore, scoreColor, {
             isAggregated: true
-          }) : averageScore !== void 0 ? averageScore.toFixed(2) : "n/a";
+          }) : averageScore !== void 0 ? tcStdDev !== void 0 && tc.isAggregated ? `${averageScore.toFixed(2)} \xB1 ${tcStdDev.toFixed(2)}` : averageScore.toFixed(2) : "n/a";
           return /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { children: [
             /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: tc.passed ? "green" : "red", children: tc.passed ? "PASS" : "FAIL" }),
             /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
@@ -1804,13 +1927,26 @@ function RunView({
 }
 
 // src/cli-simple/run.ts
+function sampleStdDev2(sum, sumSq, n) {
+  if (n < 2)
+    return void 0;
+  const mean = sum / n;
+  const variance = (sumSq - n * mean * mean) / (n - 1);
+  return variance > 0 ? Math.sqrt(variance) : 0;
+}
 function buildTestCaseSummaries(byId) {
   const summaries = [];
   for (const { name, events } of byId.values()) {
     const passed = events.every((e) => e.passed);
     const durationMs = events.reduce((sum, e) => sum + e.durationMs, 0);
     const isAggregated = events.length > 1;
-    const numericScores = [];
+    const allScores = events.flatMap(
+      (ev) => ev.evaluatorScores.map((es) => toNumericScoreFromScores(es.scores)).filter((n) => n !== void 0)
+    );
+    const averageScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : void 0;
+    const sumSq = allScores.length > 0 ? allScores.reduce((s, v) => s + v * v, 0) : 0;
+    const total = allScores.reduce((a, b) => a + b, 0);
+    const stdDev = sampleStdDev2(total, sumSq, allScores.length);
     let firstAggregatedScore;
     for (const evaluatorScores of events[0]?.evaluatorScores ?? []) {
       const scoreIdToItems = /* @__PURE__ */ new Map();
@@ -1826,21 +1962,18 @@ function buildTestCaseSummaries(byId) {
       }
       for (const items of scoreIdToItems.values()) {
         const agg = aggregateScoreItems(items);
-        if (agg) {
-          const n = toNumericScoreFromScores([agg]);
-          if (n !== void 0) {
-            numericScores.push(n);
-            if (firstAggregatedScore === void 0) {
-              firstAggregatedScore = agg;
-            }
-          }
+        if (agg && firstAggregatedScore === void 0) {
+          firstAggregatedScore = agg;
+          break;
         }
       }
+      if (firstAggregatedScore !== void 0)
+        break;
     }
-    const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
     summaries.push({
       name,
       averageScore,
+      stdDev: stdDev ?? void 0,
       aggregatedScoreItem: firstAggregatedScore,
       isAggregated,
       durationMs,
@@ -1871,12 +2004,36 @@ function scoreToColor(score) {
   }
   return ansi2.red;
 }
-function getEvaluatorSummaryLine(evaluatorName, aggregate) {
-  if (!aggregate || aggregate.count === 0) {
-    return `- ${evaluatorName.padEnd(28)} no numeric scores`;
+function getEvaluatorSummaryLines(evaluatorId, evaluatorName, aggregate, scoreItemsByKey) {
+  const lines = [];
+  const scoreKeys = [...scoreItemsByKey.keys()].filter(
+    (k) => k.startsWith(`${evaluatorId}:`)
+  );
+  if (scoreKeys.length === 0) {
+    lines.push(`- ${evaluatorName.padEnd(28)} no scores`);
+    return lines;
+  }
+  const passedFailed = aggregate != null ? ` passed=${aggregate.passed} failed=${aggregate.failed}` : "";
+  const scoreLines = [];
+  for (const key of scoreKeys) {
+    const items = scoreItemsByKey.get(key) ?? [];
+    const agg = aggregateScoreItems(items);
+    if (!agg)
+      continue;
+    const def = getScoreById(agg.id);
+    const label = def ? def.name ?? def.id : agg.id;
+    const formatted = def?.format(agg.data, { isAggregated: true }) ?? "n/a";
+    const numeric = toNumericScore(agg.data);
+    const colored = numeric !== void 0 ? colorize(formatted, scoreToColor(numeric)) : formatted;
+    scoreLines.push(`    ${label}: ${colored}`);
+  }
+  if (scoreLines.length > 0) {
+    lines.push(`- ${evaluatorName.padEnd(28)}${passedFailed}`);
+    lines.push(...scoreLines);
+  } else {
+    lines.push(`- ${evaluatorName.padEnd(28)} no numeric scores${passedFailed}`);
   }
-  const mean = aggregate.total / aggregate.count;
-  return `- ${evaluatorName.padEnd(28)} avg=${colorize(mean.toFixed(2), scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;
+  return lines;
 }
 function createBar2(value, max = 100, width = 20) {
   const safe = Math.max(0, Math.min(max, value));
@@ -1928,46 +2085,8 @@ function aggregateEvaluatorScoresFromEvents(events, evaluatorNameById) {
 }
 function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   const passLabel = passed ? colorize("PASS", `${ansi2.bold}${ansi2.green}`) : colorize("FAIL", `${ansi2.bold}${ansi2.red}`);
-  const scoreParts = [];
-  for (const item of scores) {
-    const def = getScoreById(item.id);
-    if (!def) {
-      const numeric = toNumericScore(item.data);
-      scoreParts.push(
-        numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a"
-      );
-      continue;
-    }
-    const formatted = def.format(item.data, options);
-    switch (def.displayStrategy) {
-      case "bar": {
-        const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
-        if (typeof numeric === "number" && Number.isFinite(numeric)) {
-          scoreParts.push(
-            `${colorize(formatted, scoreToColor(numeric))} ${colorize(createBar2(numeric), ansi2.dim)}`
-          );
-        } else {
-          scoreParts.push(formatted);
-        }
-        break;
-      }
-      case "number":
-        scoreParts.push(formatted);
-        break;
-      case "passFail":
-        scoreParts.push(
-          colorize(
-            formatted,
-            item.passed === true ? `${ansi2.bold}${ansi2.green}` : item.passed === false ? `${ansi2.bold}${ansi2.red}` : ansi2.dim
-          )
-        );
-        break;
-    }
-  }
-  const scoreStr = scoreParts.length > 0 ? scoreParts.join(" ") : "n/a";
-  let line = `   ${name}: ${passLabel} ${scoreStr}`;
+  const metricParts = [];
   if (metrics && metrics.length > 0) {
-    const metricParts = [];
     for (const { id, data } of metrics) {
       const def = getMetricById(id);
       if (def) {
@@ -1977,11 +2096,49 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
         );
       }
     }
-    if (metricParts.length > 0) {
-      line += ` ${metricParts.join(" ")}`;
+  }
+  const scoreLines = [];
+  for (const item of scores) {
+    const def = getScoreById(item.id);
+    const scoreLabel = def ? def.name ?? def.id : item.id;
+    let formatted;
+    if (!def) {
+      const numeric = toNumericScore(item.data);
+      formatted = numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a";
+    } else {
+      const raw = def.format(item.data, options);
+      switch (def.displayStrategy) {
+        case "bar": {
+          const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
+          if (typeof numeric === "number" && Number.isFinite(numeric)) {
+            formatted = `${colorize(raw, scoreToColor(numeric))} ${colorize(createBar2(numeric), ansi2.dim)}`;
+          } else {
+            formatted = raw;
+          }
+          break;
+        }
+        case "number":
+          formatted = raw;
+          break;
+        case "passFail":
+          formatted = colorize(
+            raw,
+            item.passed === true ? `${ansi2.bold}${ansi2.green}` : item.passed === false ? `${ansi2.bold}${ansi2.red}` : ansi2.dim
+          );
+          break;
+      }
     }
+    scoreLines.push(`      ${scoreLabel}: ${formatted}`);
   }
-  return line;
+  const lines = [];
+  const metricStr = metricParts.length > 0 ? ` ${metricParts.join(" ")}` : "";
+  lines.push(`   ${name}: ${passLabel}${metricStr}`);
+  if (scoreLines.length > 0) {
+    lines.push(...scoreLines);
+  } else {
+    lines.push(`      n/a`);
+  }
+  return lines;
 }
 async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern) {
   const dataset = await runner.resolveDatasetByName(datasetName);
@@ -2004,8 +2161,10 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
   );
   const aggregates = /* @__PURE__ */ new Map();
+  const scoreItemsByEvaluatorScore = /* @__PURE__ */ new Map();
   const testCaseByTestId = /* @__PURE__ */ new Map();
   let overallScoreTotal = 0;
+  let overallScoreSumSq = 0;
   let overallScoreCount = 0;
   let completedCount = 0;
   let totalCount = 0;
@@ -2062,19 +2221,28 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
           if (numeric !== void 0) {
             const current = aggregates.get(item.evaluatorId) ?? {
               total: 0,
+              sumSq: 0,
               count: 0,
               passed: 0,
               failed: 0
             };
             aggregates.set(item.evaluatorId, {
               total: current.total + numeric,
+              sumSq: current.sumSq + numeric * numeric,
               count: current.count + 1,
               passed: current.passed + (item.passed ? 1 : 0),
               failed: current.failed + (item.passed ? 0 : 1)
             });
             overallScoreTotal += numeric;
+            overallScoreSumSq += numeric * numeric;
             overallScoreCount += 1;
           }
+          for (const s of item.scores) {
+            const key = `${item.evaluatorId}:${s.id}`;
+            const list = scoreItemsByEvaluatorScore.get(key) ?? [];
+            list.push(s);
+            scoreItemsByEvaluatorScore.set(key, list);
+          }
         }
         const isSameTestCase = lastPrintedTestCaseId === testCaseId;
         const isLastRerun = event.rerunIndex >= event.rerunTotal;
@@ -2098,7 +2266,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         for (const item of aggregatedScores) {
           const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
           lines.push(
-            formatEvaluatorScoreLine(
+            ...formatEvaluatorScoreLine(
               name,
               item.scores,
               item.passed,
@@ -2180,18 +2348,30 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   );
   if (overallScoreCount > 0) {
     const overallAverage = overallScoreTotal / overallScoreCount;
+    const overallSd = sampleStdDev2(
+      overallScoreTotal,
+      overallScoreSumSq,
+      overallScoreCount
+    );
+    const avgStr = overallSd !== void 0 ? `${overallAverage.toFixed(2)} \xB1 ${overallSd.toFixed(2)}` : overallAverage.toFixed(2);
     console.log(
       `- overall avg score: ${colorize(
-        overallAverage.toFixed(2),
+        avgStr,
         scoreToColor(overallAverage)
       )} ${colorize(createBar2(overallAverage), ansi2.dim)}`
     );
   }
   console.log(colorize("- evaluator averages:", ansi2.magenta));
   for (const [evaluatorId, evaluatorName] of evaluatorNameById.entries()) {
-    console.log(
-      getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId))
+    const evaluatorLines = getEvaluatorSummaryLines(
+      evaluatorId,
+      evaluatorName,
+      aggregates.get(evaluatorId),
+      scoreItemsByEvaluatorScore
     );
+    for (const line of evaluatorLines) {
+      console.log(line);
+    }
   }
   const testCaseSummaries = buildTestCaseSummaries(testCaseByTestId);
   if (testCaseSummaries.length > 0) {
@@ -2207,7 +2387,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       const scoreLabel = summary.isAggregated && summary.aggregatedScoreItem ? getScoreById(summary.aggregatedScoreItem.id)?.format(
         summary.aggregatedScoreItem.data,
         { isAggregated: true }
-      ) ?? summary.averageScore.toFixed(2) : summary.averageScore.toFixed(2);
+      ) ?? summary.averageScore.toFixed(2) : summary.stdDev !== void 0 && summary.isAggregated ? `${summary.averageScore.toFixed(2)} \xB1 ${summary.stdDev.toFixed(2)}` : summary.averageScore.toFixed(2);
       console.log(
         `  ${status} ${summary.name.padEnd(24)} score=${colorize(
           scoreLabel,