@m4trix/evals 0.19.0 → 0.20.0

package/dist/cli-simple.js

@@ -268,6 +268,8 @@ function createDiffString(expected, actual, diffOptions) {
 function formatLogMessage(msg) {
   if (typeof msg === "string")
     return msg;
+  if (msg instanceof Error)
+    return msg.stack ?? msg.message;
   try {
     if (msg !== null && typeof msg === "object") {
       return JSON.stringify(msg, null, 2);
@@ -607,6 +609,7 @@ function toNumericScore(value) {
 }
 
 // src/runner/execution.ts
+var evaluatorErrorLogEntryKey = "__m4trixEvaluatorLogEntry";
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -663,20 +666,26 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
         if (!evaluateFn) {
           continue;
         }
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
+        const log = (message, options) => {
+          logs.push(createLogEntry(message, options));
+        };
+        const createError = (message, options) => {
+          const entry = createLogEntry(message, options);
+          const error = message instanceof Error ? message : new Error(entry.message);
+          error[evaluatorErrorLogEntryKey] = entry;
+          return error;
+        };
         try {
-          const logs = [];
-          const logDiff = (expected, actual, options) => {
-            logs.push(createDiffLogEntry(expected, actual, options));
-          };
-          const log = (message, options) => {
-            logs.push(createLogEntry(message, options));
-          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
           const result = yield* Effect.promise(
-            () => Promise.resolve(
-              evaluateFn({
+            () => Promise.resolve().then(
+              () => evaluateFn({
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
@@ -686,10 +695,24 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                   datasetId: task.datasetId
                 },
                 logDiff,
-                log
+                log,
+                createError
               })
             )
           );
+          if (result instanceof Error) {
+            const evaluatorError = result;
+            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(result));
+            testCaseError = result.message;
+            evaluatorScores.push({
+              evaluatorId,
+              scores: [],
+              passed: false,
+              logs: logs.length > 0 ? logs : void 0
+            });
+            continue;
+          }
           const { scores, metrics } = normalizeResult(result);
           const passed2 = computeEvaluatorPassed(evaluator, result, scores);
           evaluatorScores.push({
@@ -700,11 +723,16 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
             logs: logs.length > 0 ? logs : void 0
           });
         } catch (error) {
+          if (error instanceof Error) {
+            const taggedEntry = error[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(error));
+          }
           testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
           evaluatorScores.push({
             evaluatorId,
             scores: [],
-            passed: false
+            passed: false,
+            logs: logs.length > 0 ? logs : void 0
           });
         }
       }
@@ -1688,6 +1716,7 @@ function RunView({
               rerunTotal: event.rerunTotal,
               durationMs: events.reduce((s, e) => s + e.durationMs, 0),
               passed: events.every((e) => e.passed),
+              errorMessage: event.errorMessage,
               events,
               aggregatedEvaluatorScores,
               isAggregated
@@ -1798,8 +1827,13 @@ function RunView({
           " (",
           tc.durationMs,
           "ms)"
-        ] })
+        ] }),
+        tc.errorMessage ? /* @__PURE__ */ jsxs(Text, { color: "red", bold: true, children: [
+          " ",
+          "ERROR"
+        ] }) : null
       ] }),
+      tc.errorMessage ? /* @__PURE__ */ jsx(Text, { color: "red", children: tc.errorMessage }) : null,
       tc.aggregatedEvaluatorScores.map((item) => /* @__PURE__ */ jsxs(
         Box,
         {
@@ -2337,9 +2371,13 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
           0
         );
         const lines = [];
+        const statusSuffix = event.errorMessage ? ` ${colorize("ERROR", `${ansi2.bold}${ansi2.red}`)}` : "";
         lines.push(
-          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}`
+          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}${statusSuffix}`
         );
+        if (event.errorMessage) {
+          lines.push(colorize(event.errorMessage, ansi2.red));
+        }
         for (const item of aggregatedScores) {
           const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
           lines.push(