@m4trix/evals 0.13.0 → 0.14.0

package/dist/index.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
 var effect = require('effect');
-var jsonDiff = require('json-diff');
+var diff = require('diff');
 var crypto = require('crypto');
 var fs = require('fs');
 var path = require('path');
@@ -561,15 +561,28 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverage(values) {
+function aggregateAverageWithVariance(values) {
   if (values.length === 0) {
-    return { value: 0 };
+    return { value: 0, count: 0 };
   }
   const sum = values.reduce((s, v) => s + v.value, 0);
-  return { value: sum / values.length };
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
 }
 function aggregateAll(values) {
-  return { passed: values.length > 0 && values.every((v) => v.passed) };
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
 }
 function aggregateTokenCountSum(values) {
   const initial = {
@@ -623,18 +636,59 @@ var percentScore = Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
-  aggregate: aggregateAverage
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
 });
 var binaryScore = Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
   aggregate: aggregateAll
 });
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diff.diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
 function createDiffLogEntry(expected, actual, options) {
-  const diff = jsonDiff.diffString(expected, actual, { color: false });
+  const diff = createDiffString(expected, actual);
   return {
     type: "diff",
     label: options?.label,
@@ -644,8 +698,22 @@ function createDiffLogEntry(expected, actual, options) {
   };
 }
 function printJsonDiff(expected, actual, options = {}) {
-  const { color = true } = options;
-  const diff = jsonDiff.diffString(expected, actual, { color });
+  const diff = createDiffString(expected, actual);
+  if (options.color) {
+    const lines = diff.split("\n").map((line) => {
+      const trimmed = line.trimStart();
+      if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+        return `\x1B[31m${line}\x1B[0m`;
+      }
+      if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+        return `\x1B[32m${line}\x1B[0m`;
+      }
+      return line;
+    });
+    const colored = lines.join("\n");
+    console.log(colored || "(no differences)");
+    return colored;
+  }
   console.log(diff || "(no differences)");
   return diff;
 }