@m4trix/evals 0.17.0 → 0.19.0

package/dist/cli-simple.cjs

@@ -347,7 +347,11 @@ var Metric = {
       name: config.name,
       aggregate: config.aggregate,
       format: config.format,
-      make: (data) => ({ id: config.id, data })
+      make: (data, options) => ({
+        id: config.id,
+        data,
+        ...options?.name !== void 0 && { name: options.name }
+      })
     };
     registry.set(config.id, def);
     return def;
@@ -359,20 +363,107 @@ function getMetricById(id) {
 
 // src/evals/score.ts
 var registry2 = /* @__PURE__ */ new Map();
+function formatScoreData(def, data, options) {
+  return options?.isAggregated ? def.formatAggregate(data) : def.formatValue(data);
+}
+var ScoreAggregate = {
+  /** Average numeric fields. Use for scores like { value, delta }. */
+  averageFields(fields) {
+    return (values) => {
+      const count = values.length || 1;
+      const result = {};
+      for (const field of fields) {
+        result[field] = values.reduce(
+          (s, v) => s + (v[field] ?? 0),
+          0
+        ) / count;
+      }
+      return result;
+    };
+  },
+  /** Average selected numeric fields, with sample std dev tracked for `value`. */
+  averageWithVariance(fields) {
+    return (values) => {
+      const count = values.length;
+      const result = {};
+      for (const field of fields) {
+        result[field] = count === 0 ? 0 : values.reduce(
+          (sum, item) => sum + (item[field] ?? 0),
+          0
+        ) / count;
+      }
+      const valueField = "value";
+      const hasValueField = fields.includes(valueField);
+      if (count === 0) {
+        if (hasValueField) {
+          result[valueField] = 0;
+        }
+        return {
+          ...result,
+          stdDev: void 0,
+          count: 0
+        };
+      }
+      let stdDev;
+      if (hasValueField && count >= 2) {
+        const sum = values.reduce(
+          (s, v) => s + (v[valueField] ?? 0),
+          0
+        );
+        const sumSq = values.reduce(
+          (s, v) => {
+            const value = v[valueField] ?? 0;
+            return s + value * value;
+          },
+          0
+        );
+        const mean = sum / count;
+        const variance = (sumSq - count * mean * mean) / (count - 1);
+        stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+      }
+      return {
+        ...values[0],
+        ...result,
+        stdDev,
+        count
+      };
+    };
+  },
+  /** All runs must pass. Use for binary scores. */
+  all(values) {
+    const total = values.length;
+    const passedCount = values.filter((v) => v.passed).length;
+    return {
+      ...values[0],
+      passed: total > 0 && values.every((v) => v.passed),
+      passedCount,
+      totalCount: total
+    };
+  },
+  /** Take last value (no aggregation). Use when aggregation is not meaningful. */
+  last(values) {
+    return values[values.length - 1] ?? {};
+  }
+};
 var Score = {
+  aggregate: ScoreAggregate,
   of(config) {
     const def = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
-      aggregate: config.aggregate,
-      format: config.format,
+      formatValue: config.formatValue,
+      formatAggregate: config.formatAggregate,
+      aggregateValues: config.aggregateValues,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
         return {
           id: config.id,
           data,
-          ...passed !== void 0 && { passed }
+          ...passed !== void 0 && { passed },
+          ...options?.name !== void 0 && { name: options.name },
+          def
+          // Attach def so rendering/aggregation works without registry lookup
         };
       }
     };
@@ -385,29 +476,6 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverageWithVariance(values) {
-  if (values.length === 0) {
-    return { value: 0, count: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.value, 0);
-  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
-  const mean = sum / values.length;
-  let stdDev;
-  if (values.length >= 2) {
-    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
-    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
-  }
-  return { value: mean, stdDev, count: values.length };
-}
-function aggregateAll(values) {
-  const total = values.length;
-  const passedCount = values.filter((v) => v.passed).length;
-  return {
-    passed: total > 0 && values.every((v) => v.passed),
-    passedCount,
-    totalCount: total
-  };
-}
 function aggregateTokenCountSum(values) {
   const initial = {
     input: 0,
@@ -460,40 +528,59 @@ Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => {
-    if (options?.isAggregated) {
-      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
-    }
-    return data.value.toFixed(2);
-  },
-  aggregate: aggregateAverageWithVariance
+  formatValue: (data) => data.value.toFixed(2),
+  formatAggregate: (data) => data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`,
+  aggregateValues: Score.aggregate.averageWithVariance(["value"])
+});
+Score.of({
+  id: "delta",
+  name: "Delta",
+  displayStrategy: "number",
+  formatValue: (data) => `${data.value.toFixed(2)} (${data.delta >= 0 ? "+" : ""}${data.delta.toFixed(2)} vs baseline)`,
+  formatAggregate: (data) => `Avg: ${data.value.toFixed(2)} (Delta: ${data.delta >= 0 ? "+" : ""}${data.delta.toFixed(2)})`,
+  aggregateValues: Score.aggregate.averageFields(["value", "delta"])
 });
 Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => {
-    if (options?.isAggregated) {
-      const base = data.passed ? "All: PASSED" : "Some: FAILED";
-      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
-        return `${base} (${data.passedCount}/${data.totalCount})`;
-      }
-      return base;
+  formatValue: (data) => data.passed ? "PASSED" : "NOT PASSED",
+  formatAggregate: (data) => {
+    const base = data.passed ? "All: PASSED" : "Some: FAILED";
+    if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+      return `${base} (${data.passedCount}/${data.totalCount})`;
     }
-    return data.passed ? "PASSED" : "NOT PASSED";
+    return base;
   },
-  aggregate: aggregateAll
+  aggregateValues: Score.aggregate.all
 });
 
 // src/runner/score-utils.ts
+function getScoreDef(item) {
+  return item.def ?? getScoreById(item.id);
+}
+function lastNonEmptyName(items) {
+  for (let i = items.length - 1; i >= 0; i--) {
+    const n = items[i].name;
+    if (n != null && n.trim().length > 0)
+      return n;
+  }
+  return void 0;
+}
 function aggregateScoreItems(items) {
   if (items.length === 0)
     return void 0;
-  const def = getScoreById(items[0].id);
-  if (!def?.aggregate)
+  const def = getScoreDef(items[0]);
+  if (!def?.aggregateValues)
     return items[items.length - 1];
-  const aggregated = def.aggregate(items.map((i) => i.data));
-  return { ...items[0], data: aggregated };
+  const aggregated = def.aggregateValues(items.map((i) => i.data));
+  const nameOverride = lastNonEmptyName(items);
+  return {
+    ...items[0],
+    data: aggregated,
+    def,
+    ...nameOverride !== void 0 && { name: nameOverride }
+  };
 }
 function aggregateMetricItems(items) {
   if (items.length === 0)
@@ -502,11 +589,16 @@ function aggregateMetricItems(items) {
   if (!def?.aggregate)
     return items[items.length - 1];
   const aggregated = def.aggregate(items.map((i) => i.data));
-  return { ...items[0], data: aggregated };
+  const nameOverride = lastNonEmptyName(items);
+  return {
+    ...items[0],
+    data: aggregated,
+    ...nameOverride !== void 0 && { name: nameOverride }
+  };
 }
 function toNumericScoreFromScores(scores) {
   for (const item of scores) {
-    const def = getScoreById(item.id);
+    const def = getScoreDef(item);
     if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
       const value = item.data.value;
       if (typeof value === "number" && Number.isFinite(value)) {
@@ -894,7 +986,7 @@ var createPersistenceWorker = (queue) => effect.Effect.forever(
       () => appendJsonLine(message.artifactPath, {
         runId: message.runId,
         ts: Date.now(),
-        ...message.payload
+        ...typeof message.payload === "object" && message.payload !== null && !Array.isArray(message.payload) ? message.payload : {}
       })
     );
   })
@@ -1480,7 +1572,7 @@ function aggregateEvaluatorScores(events, nameById) {
       if (agg)
         aggregatedScores.push(agg);
     }
-    const aggregatedMetrics = Array.from(metricIdToItems.entries()).map(([id, items]) => aggregateMetricItems(items)).filter((m) => m !== void 0);
+    const aggregatedMetrics = Array.from(metricIdToItems.entries()).map(([, items]) => aggregateMetricItems(items)).filter((m) => m !== void 0);
     const passed = events.every((ev) => {
       const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
       return es?.passed ?? false;
@@ -1500,13 +1592,13 @@ function aggregateEvaluatorScores(events, nameById) {
   }
   return result;
 }
-function formatScorePart(item, scoreToColor2, options) {
-  const def = getScoreById(item.id);
+function formatScorePart(item, _scoreToColor, options) {
+  const def = item.def ?? getScoreById(item.id);
   if (!def) {
     const numeric = toNumericScore(item.data);
     return numeric !== void 0 ? `${numeric.toFixed(2)}` : "n/a";
   }
-  const formatted = def.format(item.data, options);
+  const formatted = formatScoreData(def, item.data, options);
   if (def.displayStrategy === "bar") {
     const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
     if (typeof numeric === "number" && Number.isFinite(numeric)) {
@@ -1564,8 +1656,6 @@ function RunView({
     const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
         if (event.type === "TestCaseProgress") {
-          const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
-          numericScores.length > 0 ? numericScores.reduce((sum, v) => sum + v, 0) / numericScores.length : void 0;
           for (const item of event.evaluatorScores) {
             const numeric = toNumericScoreFromScores(item.scores);
             if (numeric !== void 0) {
@@ -1655,16 +1745,17 @@ function RunView({
       onComplete(new Error(`Run failed: ${finalEvent.errorMessage}`));
       return;
     }
+    const completed = finalEvent;
     setSummary({
-      passedTestCases: finalEvent.passedTestCases,
-      failedTestCases: finalEvent.failedTestCases,
-      totalTestCases: finalEvent.totalTestCases,
+      passedTestCases: completed.passedTestCases,
+      failedTestCases: completed.failedTestCases,
+      totalTestCases: completed.totalTestCases,
       overallScoreTotal,
       overallScoreSumSq,
       overallScoreCount,
       aggregates: new Map(aggregates),
       scoreItemsByEvaluatorScore: new Map(scoreItemsByEvaluatorScore),
-      artifactPath: finalEvent.artifactPath
+      artifactPath: completed.artifactPath
     });
     setPhase("completed");
     setTimeout(() => onComplete(), 200);
@@ -1755,9 +1846,10 @@ function RunView({
                   const formatted = def.format(m.data, {
                     isAggregated: tc.isAggregated
                   });
+                  const label = m.name ?? def.name;
                   return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
                     "[",
-                    def.name ? `${def.name}: ` : "",
+                    label ? `${label}: ` : "",
                     formatted,
                     "]",
                     " "
@@ -1766,8 +1858,8 @@ function RunView({
               ] }) : null
             ] }),
             item.scores.length > 0 ? item.scores.map((s, idx) => {
-              const def = getScoreById(s.id);
-              const scoreLabel = def ? def.name ?? def.id : s.id;
+              const def = s.def ?? getScoreById(s.id);
+              const scoreLabel = s.name ?? def?.name ?? def?.id ?? s.id;
               return /* @__PURE__ */ jsxRuntime.jsxs(
                 ink.Text,
                 {
@@ -1870,11 +1962,9 @@ function RunView({
               const aggregated = aggregateScoreItems(items);
               if (!aggregated)
                 return null;
-              const def = getScoreById(aggregated.id);
-              const label = def ? def.name ?? def.id : aggregated.id;
-              const formatted = def?.format(aggregated.data, {
-                isAggregated: true
-              }) ?? "n/a";
+              const def = aggregated.def ?? getScoreById(aggregated.id);
+              const label = aggregated.name ?? def?.name ?? def?.id ?? aggregated.id;
+              const formatted = def ? def.formatAggregate(aggregated.data) : "n/a";
               const numeric = toNumericScore(aggregated.data);
               return /* @__PURE__ */ jsxRuntime.jsxs(
                 ink.Text,
@@ -2033,9 +2123,9 @@ function getEvaluatorSummaryLines(evaluatorId, evaluatorName, aggregate, scoreIt
     const agg = aggregateScoreItems(items);
     if (!agg)
       continue;
-    const def = getScoreById(agg.id);
-    const label = def ? def.name ?? def.id : agg.id;
-    const formatted = def?.format(agg.data, { isAggregated: true }) ?? "n/a";
+    const def = agg.def ?? getScoreById(agg.id);
+    const label = agg.name ?? def?.name ?? def?.id ?? agg.id;
+    const formatted = def ? def.formatAggregate(agg.data) : "n/a";
     const numeric = toNumericScore(agg.data);
     const colored = numeric !== void 0 ? colorize(formatted, scoreToColor(numeric)) : formatted;
     scoreLines.push(`    ${label}: ${colored}`);
@@ -2053,7 +2143,7 @@ function createBar2(value, max = 100, width = 20) {
   const filled = Math.round(safe / max * width);
   return `${"\u2588".repeat(filled)}${"\u2591".repeat(width - filled)}`;
 }
-function aggregateEvaluatorScoresFromEvents(events, evaluatorNameById) {
+function aggregateEvaluatorScoresFromEvents(events, _evaluatorNameById) {
   if (events.length === 0)
     return [];
   const evaluatorIds = new Set(
@@ -2100,26 +2190,27 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   const passLabel = passed ? colorize("PASS", `${ansi2.bold}${ansi2.green}`) : colorize("FAIL", `${ansi2.bold}${ansi2.red}`);
   const metricParts = [];
   if (metrics && metrics.length > 0) {
-    for (const { id, data } of metrics) {
-      const def = getMetricById(id);
+    for (const m of metrics) {
+      const def = getMetricById(m.id);
       if (def) {
-        const formatted = def.format(data, options);
+        const formatted = def.format(m.data, options);
+        const label = m.name ?? def.name;
         metricParts.push(
-          def.name ? `[${def.name}: ${formatted}]` : `[${formatted}]`
+          label ? `[${label}: ${formatted}]` : `[${formatted}]`
         );
       }
     }
   }
   const scoreLines = [];
   for (const item of scores) {
-    const def = getScoreById(item.id);
-    const scoreLabel = def ? def.name ?? def.id : item.id;
+    const def = item.def ?? getScoreById(item.id);
+    const scoreLabel = item.name ?? def?.name ?? def?.id ?? item.id;
     let formatted;
     if (!def) {
       const numeric = toNumericScore(item.data);
       formatted = numeric !== void 0 ? colorize(numeric.toFixed(2), scoreToColor(numeric)) : "n/a";
     } else {
-      const raw = def.format(item.data, options);
+      const raw = formatScoreData(def, item.data, options);
       switch (def.displayStrategy) {
         case "bar": {
           const numeric = typeof item.data === "object" && item.data !== null && "value" in item.data ? item.data.value : toNumericScore(item.data);
@@ -2271,7 +2362,6 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
           (s, e) => s + e.durationMs,
           0
         );
-        existing.events.every((e) => e.passed);
         const lines = [];
         lines.push(
           `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}`
@@ -2349,18 +2439,19 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   if (finalEvent.type === "RunFailed") {
     throw new Error(`Run failed: ${finalEvent.errorMessage}`);
   }
+  const completed = finalEvent;
   console.log("");
   console.log(colorize("=== Run Summary ===", `${ansi2.bold}${ansi2.cyan}`));
   console.log(
     `- passed: ${colorize(
-      `${finalEvent.passedTestCases}/${finalEvent.totalTestCases}`,
+      `${completed.passedTestCases}/${completed.totalTestCases}`,
       ansi2.green
     )}`
   );
   console.log(
     `- failed: ${colorize(
-      `${finalEvent.failedTestCases}/${finalEvent.totalTestCases}`,
-      finalEvent.failedTestCases > 0 ? ansi2.red : ansi2.dim
+      `${completed.failedTestCases}/${completed.totalTestCases}`,
+      completed.failedTestCases > 0 ? ansi2.red : ansi2.dim
     )}`
   );
   if (overallScoreCount > 0) {
@@ -2401,10 +2492,10 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         );
         continue;
       }
-      const scoreLabel = summary.isAggregated && summary.aggregatedScoreItem ? getScoreById(summary.aggregatedScoreItem.id)?.format(
-        summary.aggregatedScoreItem.data,
-        { isAggregated: true }
-      ) ?? summary.averageScore.toFixed(2) : summary.stdDev !== void 0 && summary.isAggregated ? `${summary.averageScore.toFixed(2)} \xB1 ${summary.stdDev.toFixed(2)}` : summary.averageScore.toFixed(2);
+      const scoreLabel = summary.isAggregated && summary.aggregatedScoreItem ? (() => {
+        const def = summary.aggregatedScoreItem.def ?? getScoreById(summary.aggregatedScoreItem.id);
+        return def ? def.formatAggregate(summary.aggregatedScoreItem.data) : summary.averageScore.toFixed(2);
+      })() : summary.stdDev !== void 0 && summary.isAggregated ? `${summary.averageScore.toFixed(2)} \xB1 ${summary.stdDev.toFixed(2)}` : summary.averageScore.toFixed(2);
       console.log(
         `  ${status} ${summary.name.padEnd(24)} score=${colorize(
           scoreLabel,
@@ -2413,7 +2504,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       );
     }
   }
-  console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi2.dim)}`);
+  console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
 }
 async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
   return new Promise((resolve5, reject) => {