@m4trix/evals 0.13.0 → 0.14.0

package/dist/cli-simple.js

@@ -6,7 +6,7 @@ import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffString } from 'json-diff';
+import { diffLines } from 'diff';
 import React2, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
@@ -260,8 +260,35 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
 function createDiffLogEntry(expected, actual, options) {
-  const diff = diffString(expected, actual, { color: false });
+  const diff = createDiffString(expected, actual);
   return {
     type: "diff",
     label: options?.label,
@@ -271,7 +298,7 @@ function createDiffLogEntry(expected, actual, options) {
   };
 }
 function getDiffLines(entry) {
-  const raw = diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
+  const raw = createDiffString(entry.expected, entry.actual) || "(no differences)";
   return raw.split("\n").map((line) => {
     const trimmed = line.trimStart();
     if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -331,15 +358,28 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverage(values) {
+function aggregateAverageWithVariance(values) {
   if (values.length === 0) {
-    return { value: 0 };
+    return { value: 0, count: 0 };
   }
   const sum = values.reduce((s, v) => s + v.value, 0);
-  return { value: sum / values.length };
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
 }
 function aggregateAll(values) {
-  return { passed: values.length > 0 && values.every((v) => v.passed) };
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
 }
 function aggregateTokenCountSum(values) {
   const initial = {
@@ -393,14 +433,28 @@ Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
-  aggregate: aggregateAverage
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
 });
 Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
   aggregate: aggregateAll
 });
 
@@ -1339,6 +1393,13 @@ function Spinner({ label = "Running" }) {
     label
   ] });
 }
+function sampleStdDev(sum, sumSq, n) {
+  if (n < 2)
+    return void 0;
+  const mean = sum / n;
+  const variance = (sumSq - n * mean * mean) / (n - 1);
+  return variance > 0 ? Math.sqrt(variance) : 0;
+}
 function scoreColor(score) {
   if (score >= 80)
     return "green";
@@ -1458,6 +1519,7 @@ function RunView({
     setEvaluatorNameById(nameById);
     const aggregates = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
+    let overallScoreSumSq = 0;
     let overallScoreCount = 0;
     const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
@@ -1469,17 +1531,20 @@ function RunView({
             if (numeric !== void 0) {
               const current = aggregates.get(item.evaluatorId) ?? {
                 total: 0,
+                sumSq: 0,
                 count: 0,
                 passed: 0,
                 failed: 0
               };
               aggregates.set(item.evaluatorId, {
                 total: current.total + numeric,
+                sumSq: current.sumSq + numeric * numeric,
                 count: current.count + 1,
                 passed: current.passed + (item.passed ? 1 : 0),
                 failed: current.failed + (item.passed ? 0 : 1)
               });
               overallScoreTotal += numeric;
+              overallScoreSumSq += numeric * numeric;
               overallScoreCount += 1;
             }
           }
@@ -1549,6 +1614,7 @@ function RunView({
       failedTestCases: finalEvent.failedTestCases,
       totalTestCases: finalEvent.totalTestCases,
       overallScoreTotal,
+      overallScoreSumSq,
       overallScoreCount,
       aggregates: new Map(aggregates),
       artifactPath: finalEvent.artifactPath
@@ -1706,7 +1772,14 @@ function RunView({
           label: "overall avg",
           value: summary.overallScoreTotal / summary.overallScoreCount,
           barWidth: 20,
-          format: (v) => v.toFixed(2)
+          format: (v) => {
+            const sd = sampleStdDev(
+              summary.overallScoreTotal,
+              summary.overallScoreSumSq,
+              summary.overallScoreCount
+            );
+            return sd !== void 0 ? `${v.toFixed(2)} \xB1 ${sd.toFixed(2)}` : v.toFixed(2);
+          }
         }
       ) }),
       /* @__PURE__ */ jsxs(Box, { marginTop: 1, flexDirection: "column", children: [
@@ -1721,11 +1794,13 @@ function RunView({
             ] }, id);
           }
           const mean = agg.total / agg.count;
+          const sd = sampleStdDev(agg.total, agg.sumSq, agg.count);
+          const meanStr = sd !== void 0 ? `${mean.toFixed(2)} \xB1 ${sd.toFixed(2)}` : mean.toFixed(2);
           return /* @__PURE__ */ jsxs(Text, { children: [
             "- ",
             name.padEnd(28),
             " avg=",
-            /* @__PURE__ */ jsx(Text, { color: scoreColor(mean), children: mean.toFixed(2) }),
+            /* @__PURE__ */ jsx(Text, { color: scoreColor(mean), children: meanStr }),
             " ",
             "passed=",
             agg.passed,
@@ -1737,14 +1812,17 @@ function RunView({
       /* @__PURE__ */ jsxs(Box, { marginTop: 1, flexDirection: "column", children: [
         /* @__PURE__ */ jsx(Text, { color: "magenta", children: "test case scores" }),
         testCases.map((tc) => {
-          const numericScores = tc.aggregatedEvaluatorScores.flatMap(
-            (item) => item.scores.map((s) => toNumericScoreFromScores([s])).filter((n) => n !== void 0)
+          const allScores = tc.events.flatMap(
+            (ev) => ev.evaluatorScores.map((es) => toNumericScoreFromScores(es.scores)).filter((n) => n !== void 0)
           );
-          const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
+          const averageScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : void 0;
+          const sumSq = allScores.length > 0 ? allScores.reduce((s, v) => s + v * v, 0) : 0;
+          const total = allScores.reduce((a, b) => a + b, 0);
+          const tcStdDev = sampleStdDev(total, sumSq, allScores.length);
           const firstScore = tc.aggregatedEvaluatorScores[0]?.scores[0];
           const scoreLabel = firstScore && tc.isAggregated ? formatScorePart(firstScore, scoreColor, {
             isAggregated: true
-          }) : averageScore !== void 0 ? averageScore.toFixed(2) : "n/a";
+          }) : averageScore !== void 0 ? tcStdDev !== void 0 && tc.isAggregated ? `${averageScore.toFixed(2)} \xB1 ${tcStdDev.toFixed(2)}` : averageScore.toFixed(2) : "n/a";
           return /* @__PURE__ */ jsxs(Box, { children: [
             /* @__PURE__ */ jsx(Text, { color: tc.passed ? "green" : "red", children: tc.passed ? "PASS" : "FAIL" }),
             /* @__PURE__ */ jsxs(Text, { children: [
@@ -1778,13 +1856,26 @@ function RunView({
 }
 
 // src/cli-simple/run.ts
+function sampleStdDev2(sum, sumSq, n) {
+  if (n < 2)
+    return void 0;
+  const mean = sum / n;
+  const variance = (sumSq - n * mean * mean) / (n - 1);
+  return variance > 0 ? Math.sqrt(variance) : 0;
+}
 function buildTestCaseSummaries(byId) {
   const summaries = [];
   for (const { name, events } of byId.values()) {
     const passed = events.every((e) => e.passed);
     const durationMs = events.reduce((sum, e) => sum + e.durationMs, 0);
     const isAggregated = events.length > 1;
-    const numericScores = [];
+    const allScores = events.flatMap(
+      (ev) => ev.evaluatorScores.map((es) => toNumericScoreFromScores(es.scores)).filter((n) => n !== void 0)
+    );
+    const averageScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : void 0;
+    const sumSq = allScores.length > 0 ? allScores.reduce((s, v) => s + v * v, 0) : 0;
+    const total = allScores.reduce((a, b) => a + b, 0);
+    const stdDev = sampleStdDev2(total, sumSq, allScores.length);
     let firstAggregatedScore;
     for (const evaluatorScores of events[0]?.evaluatorScores ?? []) {
       const scoreIdToItems = /* @__PURE__ */ new Map();
@@ -1800,21 +1891,18 @@ function buildTestCaseSummaries(byId) {
       }
       for (const items of scoreIdToItems.values()) {
         const agg = aggregateScoreItems(items);
-        if (agg) {
-          const n = toNumericScoreFromScores([agg]);
-          if (n !== void 0) {
-            numericScores.push(n);
-            if (firstAggregatedScore === void 0) {
-              firstAggregatedScore = agg;
-            }
-          }
+        if (agg && firstAggregatedScore === void 0) {
+          firstAggregatedScore = agg;
+          break;
         }
       }
+      if (firstAggregatedScore !== void 0)
+        break;
     }
-    const averageScore = numericScores.length > 0 ? numericScores.reduce((a, b) => a + b, 0) / numericScores.length : void 0;
     summaries.push({
       name,
       averageScore,
+      stdDev: stdDev ?? void 0,
       aggregatedScoreItem: firstAggregatedScore,
       isAggregated,
       durationMs,
@@ -1850,7 +1938,9 @@ function getEvaluatorSummaryLine(evaluatorName, aggregate) {
     return `- ${evaluatorName.padEnd(28)} no numeric scores`;
   }
   const mean = aggregate.total / aggregate.count;
-  return `- ${evaluatorName.padEnd(28)} avg=${colorize(mean.toFixed(2), scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;
+  const sd = sampleStdDev2(aggregate.total, aggregate.sumSq, aggregate.count);
+  const meanStr = sd !== void 0 ? `${mean.toFixed(2)} \xB1 ${sd.toFixed(2)}` : mean.toFixed(2);
+  return `- ${evaluatorName.padEnd(28)} avg=${colorize(meanStr, scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;
 }
 function createBar2(value, max = 100, width = 20) {
   const safe = Math.max(0, Math.min(max, value));
@@ -1980,6 +2070,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   const aggregates = /* @__PURE__ */ new Map();
   const testCaseByTestId = /* @__PURE__ */ new Map();
   let overallScoreTotal = 0;
+  let overallScoreSumSq = 0;
   let overallScoreCount = 0;
   let completedCount = 0;
   let totalCount = 0;
@@ -2036,17 +2127,20 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
           if (numeric !== void 0) {
             const current = aggregates.get(item.evaluatorId) ?? {
               total: 0,
+              sumSq: 0,
               count: 0,
               passed: 0,
               failed: 0
             };
             aggregates.set(item.evaluatorId, {
               total: current.total + numeric,
+              sumSq: current.sumSq + numeric * numeric,
               count: current.count + 1,
               passed: current.passed + (item.passed ? 1 : 0),
               failed: current.failed + (item.passed ? 0 : 1)
             });
             overallScoreTotal += numeric;
+            overallScoreSumSq += numeric * numeric;
             overallScoreCount += 1;
           }
         }
@@ -2154,9 +2248,15 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   );
   if (overallScoreCount > 0) {
     const overallAverage = overallScoreTotal / overallScoreCount;
+    const overallSd = sampleStdDev2(
+      overallScoreTotal,
+      overallScoreSumSq,
+      overallScoreCount
+    );
+    const avgStr = overallSd !== void 0 ? `${overallAverage.toFixed(2)} \xB1 ${overallSd.toFixed(2)}` : overallAverage.toFixed(2);
     console.log(
       `- overall avg score: ${colorize(
-        overallAverage.toFixed(2),
+        avgStr,
         scoreToColor(overallAverage)
       )} ${colorize(createBar2(overallAverage), ansi2.dim)}`
     );
@@ -2181,7 +2281,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       const scoreLabel = summary.isAggregated && summary.aggregatedScoreItem ? getScoreById(summary.aggregatedScoreItem.id)?.format(
         summary.aggregatedScoreItem.data,
         { isAggregated: true }
-      ) ?? summary.averageScore.toFixed(2) : summary.averageScore.toFixed(2);
+      ) ?? summary.averageScore.toFixed(2) : summary.stdDev !== void 0 && summary.isAggregated ? `${summary.averageScore.toFixed(2)} \xB1 ${summary.stdDev.toFixed(2)}` : summary.averageScore.toFixed(2);
       console.log(
         `  ${status} ${summary.name.padEnd(24)} score=${colorize(
           scoreLabel,