@m4trix/evals 0.18.0 → 0.20.0

package/dist/index.d.ts

@@ -257,8 +257,15 @@ interface EvaluateArgs<TInput, TOutput = unknown, TCtx = Record<string, never>>
     log: (message: unknown, options?: {
         label?: string;
     }) => void;
+    /**
+     * Creates an Error from string/object payloads for `return createError(...)` (or `throw createError(...)`).
+     * The payload is also logged and shown by the CLI when the evaluator fails.
+     */
+    createError: (message: unknown, options?: {
+        label?: string;
+    }) => Error;
 }
-type EvaluateFn<TInput, TOutput, TScore, TCtx> = (args: EvaluateArgs<TInput, TOutput, TCtx>) => TScore | Promise<TScore>;
+type EvaluateFn<TInput, TOutput, TScore, TCtx> = (args: EvaluateArgs<TInput, TOutput, TCtx>) => TScore | Error | Promise<TScore | Error>;
 interface EvaluatorDefineConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any, TS extends Schema.Schema.Any> {
     name: string;
     inputSchema: TI;
@@ -289,6 +296,8 @@ declare class Evaluator<TInput = unknown, TOutput = unknown, TScore = unknown, T
 interface MetricItem<TData = unknown> {
     readonly id: string;
     readonly data: TData;
+    /** Per-item display name override (wins over def.name in rendering) */
+    readonly name?: string;
 }
 interface FormatMetricOptions {
     isAggregated?: boolean;
@@ -298,7 +307,9 @@ interface MetricDef<TData = unknown> {
     readonly name?: string;
     readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
     format(data: TData, options?: FormatMetricOptions): string;
-    make(data: TData): MetricItem<TData>;
+    make(data: TData, options?: {
+        name?: string;
+    }): MetricItem<TData>;
 }
 declare const Metric: {
     of<TData>(config: {
@@ -315,6 +326,8 @@ interface ScoreItem<TData = unknown> {
     readonly id: string;
     readonly data: TData;
     readonly passed?: boolean;
+    /** Per-item display name override (wins over def.name in rendering) */
+    readonly name?: string;
     /** Attached def for formatting/aggregation without registry lookup (avoids n/a across module boundaries) */
     readonly def?: ScoreDef<TData>;
 }
@@ -330,6 +343,7 @@ interface ScoreDef<TData = unknown> {
     readonly aggregateValues: (values: ReadonlyArray<TData>) => TData;
     make(data: TData, options?: {
         definePassed?: (data: TData) => boolean;
+        name?: string;
     }): ScoreItem<TData>;
 }
 /** Helper to format using the right method based on isAggregated (for consumers that need a single entry point) */
@@ -338,22 +352,20 @@ declare const Score: {
     aggregate: {
         /** Average numeric fields. Use for scores like { value, delta }. */
         averageFields<K extends string>(fields: readonly K[]): (values: readonly Record<K, number>[]) => Record<K, number>;
-        /** Average `value` with sample std dev. Use for percent-style scores. */
-        averageWithVariance<T extends {
-            value: number;
-        }>(values: readonly T[]): T & {
+        /** Average selected numeric fields, with sample std dev tracked for `value`. */
+        averageWithVariance<K_1 extends string>(fields: readonly K_1[]): (values: readonly Record<K_1, number>[]) => Record<K_1, number> & {
             stdDev?: number | undefined;
             count: number;
         };
         /** All runs must pass. Use for binary scores. */
-        all<T_1 extends {
+        all<T extends {
             passed: boolean;
-        }>(values: readonly T_1[]): T_1 & {
+        }>(values: readonly T[]): T & {
             passedCount?: number | undefined;
             totalCount?: number | undefined;
         };
         /** Take last value (no aggregation). Use when aggregation is not meaningful. */
-        last<T_2>(values: readonly T_2[]): T_2;
+        last<T_1>(values: readonly T_1[]): T_1;
     };
     of<TData>(config: {
         id: string;

package/dist/index.js

@@ -501,7 +501,11 @@ var Metric = {
       name: config.name,
       aggregate: config.aggregate,
       format: config.format,
-      make: (data) => ({ id: config.id, data })
+      make: (data, options) => ({
+        id: config.id,
+        data,
+        ...options?.name !== void 0 && { name: options.name }
+      })
     };
     registry.set(config.id, def);
     return def;
@@ -523,25 +527,61 @@ var ScoreAggregate = {
       const count = values.length || 1;
       const result = {};
       for (const field of fields) {
-        result[field] = values.reduce((s, v) => s + (v[field] ?? 0), 0) / count;
+        result[field] = values.reduce(
+          (s, v) => s + (v[field] ?? 0),
+          0
+        ) / count;
       }
       return result;
     };
   },
-  /** Average `value` with sample std dev. Use for percent-style scores. */
-  averageWithVariance(values) {
-    if (values.length === 0) {
-      return { value: 0, stdDev: void 0, count: 0 };
-    }
-    const sum = values.reduce((s, v) => s + v.value, 0);
-    const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
-    const mean = sum / values.length;
-    let stdDev;
-    if (values.length >= 2) {
-      const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
-      stdDev = variance > 0 ? Math.sqrt(variance) : 0;
-    }
-    return { ...values[0], value: mean, stdDev, count: values.length };
+  /** Average selected numeric fields, with sample std dev tracked for `value`. */
+  averageWithVariance(fields) {
+    return (values) => {
+      const count = values.length;
+      const result = {};
+      for (const field of fields) {
+        result[field] = count === 0 ? 0 : values.reduce(
+          (sum, item) => sum + (item[field] ?? 0),
+          0
+        ) / count;
+      }
+      const valueField = "value";
+      const hasValueField = fields.includes(valueField);
+      if (count === 0) {
+        if (hasValueField) {
+          result[valueField] = 0;
+        }
+        return {
+          ...result,
+          stdDev: void 0,
+          count: 0
+        };
+      }
+      let stdDev;
+      if (hasValueField && count >= 2) {
+        const sum = values.reduce(
+          (s, v) => s + (v[valueField] ?? 0),
+          0
+        );
+        const sumSq = values.reduce(
+          (s, v) => {
+            const value = v[valueField] ?? 0;
+            return s + value * value;
+          },
+          0
+        );
+        const mean = sum / count;
+        const variance = (sumSq - count * mean * mean) / (count - 1);
+        stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+      }
+      return {
+        ...values[0],
+        ...result,
+        stdDev,
+        count
+      };
+    };
   },
   /** All runs must pass. Use for binary scores. */
   all(values) {
@@ -575,6 +615,7 @@ var Score = {
           id: config.id,
           data,
           ...passed !== void 0 && { passed },
+          ...options?.name !== void 0 && { name: options.name },
           def
           // Attach def so rendering/aggregation works without registry lookup
         };
@@ -643,7 +684,7 @@ var percentScore = Score.of({
   displayStrategy: "bar",
   formatValue: (data) => data.value.toFixed(2),
   formatAggregate: (data) => data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`,
-  aggregateValues: Score.aggregate.averageWithVariance
+  aggregateValues: Score.aggregate.averageWithVariance(["value"])
 });
 var deltaScore = Score.of({
   id: "delta",
@@ -675,6 +716,8 @@ function createDiffString(expected, actual, diffOptions) {
 function formatLogMessage(msg) {
   if (typeof msg === "string")
     return msg;
+  if (msg instanceof Error)
+    return msg.stack ?? msg.message;
   try {
     if (msg !== null && typeof msg === "object") {
       return JSON.stringify(msg, null, 2);
@@ -1021,6 +1064,7 @@ function toNumericScore(value) {
 }
 
 // src/runner/execution.ts
+var evaluatorErrorLogEntryKey = "__m4trixEvaluatorLogEntry";
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -1077,20 +1121,26 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
         if (!evaluateFn) {
           continue;
         }
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
+        const log = (message, options) => {
+          logs.push(createLogEntry(message, options));
+        };
+        const createError = (message, options) => {
+          const entry = createLogEntry(message, options);
+          const error = message instanceof Error ? message : new Error(entry.message);
+          error[evaluatorErrorLogEntryKey] = entry;
+          return error;
+        };
         try {
-          const logs = [];
-          const logDiff = (expected, actual, options) => {
-            logs.push(createDiffLogEntry(expected, actual, options));
-          };
-          const log = (message, options) => {
-            logs.push(createLogEntry(message, options));
-          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
           const result = yield* Effect.promise(
-            () => Promise.resolve(
-              evaluateFn({
+            () => Promise.resolve().then(
+              () => evaluateFn({
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
@@ -1100,10 +1150,24 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                   datasetId: task.datasetId
                 },
                 logDiff,
-                log
+                log,
+                createError
               })
             )
           );
+          if (result instanceof Error) {
+            const evaluatorError = result;
+            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(result));
+            testCaseError = result.message;
+            evaluatorScores.push({
+              evaluatorId,
+              scores: [],
+              passed: false,
+              logs: logs.length > 0 ? logs : void 0
+            });
+            continue;
+          }
           const { scores, metrics } = normalizeResult(result);
           const passed2 = computeEvaluatorPassed(evaluator, result, scores);
           evaluatorScores.push({
@@ -1114,11 +1178,16 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
             logs: logs.length > 0 ? logs : void 0
           });
         } catch (error) {
+          if (error instanceof Error) {
+            const taggedEntry = error[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(error));
+          }
           testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
           evaluatorScores.push({
             evaluatorId,
             scores: [],
-            passed: false
+            passed: false,
+            logs: logs.length > 0 ? logs : void 0
           });
         }
       }