@m4trix/evals 0.13.0 → 0.14.0

package/dist/index.d.ts

@@ -74,6 +74,7 @@ interface CliState {
     datasetMenuIndex: number;
     runMenuIndex: number;
     detailsScrollOffset: number;
+    overviewScrollOffset: number;
     selectedEvaluatorIds: string[];
     evaluatorMenuIndex: number;
     searchQuery: string;
@@ -412,10 +413,14 @@ declare const latencyMetric: MetricDef<LatencyData>;
 
 interface PercentScoreData {
     value: number;
+    stdDev?: number;
+    count?: number;
 }
 declare const percentScore: ScoreDef<PercentScoreData>;
 interface BinaryScoreData {
     passed: boolean;
+    passedCount?: number;
+    totalCount?: number;
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 export { Schema as S } from 'effect';
-import { diffString } from 'json-diff';
+import { diffLines } from 'diff';
 import { randomUUID } from 'crypto';
 import { existsSync } from 'fs';
 import { resolve as resolve$1, relative, join, dirname } from 'path';
@@ -539,15 +539,28 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverage(values) {
+function aggregateAverageWithVariance(values) {
   if (values.length === 0) {
-    return { value: 0 };
+    return { value: 0, count: 0 };
   }
   const sum = values.reduce((s, v) => s + v.value, 0);
-  return { value: sum / values.length };
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
 }
 function aggregateAll(values) {
-  return { passed: values.length > 0 && values.every((v) => v.passed) };
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
 }
 function aggregateTokenCountSum(values) {
   const initial = {
@@ -601,18 +614,59 @@ var percentScore = Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
-  aggregate: aggregateAverage
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
 });
 var binaryScore = Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
   aggregate: aggregateAll
 });
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
 function createDiffLogEntry(expected, actual, options) {
-  const diff = diffString(expected, actual, { color: false });
+  const diff = createDiffString(expected, actual);
   return {
     type: "diff",
     label: options?.label,
@@ -622,8 +676,22 @@ function createDiffLogEntry(expected, actual, options) {
   };
 }
 function printJsonDiff(expected, actual, options = {}) {
-  const { color = true } = options;
-  const diff = diffString(expected, actual, { color });
+  const diff = createDiffString(expected, actual);
+  if (options.color) {
+    const lines = diff.split("\n").map((line) => {
+      const trimmed = line.trimStart();
+      if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+        return `\x1B[31m${line}\x1B[0m`;
+      }
+      if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+        return `\x1B[32m${line}\x1B[0m`;
+      }
+      return line;
+    });
+    const colored = lines.join("\n");
+    console.log(colored || "(no differences)");
+    return colored;
+  }
   console.log(diff || "(no differences)");
   return diff;
 }