@m4trix/evals 0.19.0 → 0.21.0

package/dist/cli-simple.js

@@ -7,7 +7,8 @@ import * as jitiModule from 'jiti';
 import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
 import { diffString } from 'json-diff';
-import React2, { useState, useEffect, useCallback } from 'react';
+import * as React2 from 'react';
+import React2__default, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
 
@@ -268,6 +269,8 @@ function createDiffString(expected, actual, diffOptions) {
 function formatLogMessage(msg) {
   if (typeof msg === "string")
     return msg;
+  if (msg instanceof Error)
+    return msg.stack ?? msg.message;
   try {
     if (msg !== null && typeof msg === "object") {
       return JSON.stringify(msg, null, 2);
@@ -607,6 +610,7 @@ function toNumericScore(value) {
 }
 
 // src/runner/execution.ts
+var evaluatorErrorLogEntryKey = "__m4trixEvaluatorLogEntry";
 function computeEvaluatorPassed(evaluator, result, scores) {
   const scoresWithPassed = scores.filter((s) => "passed" in s && s.passed !== void 0);
   if (scoresWithPassed.length > 0) {
@@ -648,13 +652,27 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
   return Effect.gen(function* () {
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
       const evaluatorRunId = `run-${randomUUID()}`;
       const started = Date.now();
+      const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      yield* publishEvent({
+        type: "TestCaseStarted",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        startedTestCases: startedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns
+      });
       const evaluatorScores = [];
       let testCaseError;
       const output = readOutput(testCaseItem.testCase);
@@ -663,20 +681,26 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
         if (!evaluateFn) {
           continue;
         }
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
+        const log = (message, options) => {
+          logs.push(createLogEntry(message, options));
+        };
+        const createError = (message, options) => {
+          const entry = createLogEntry(message, options);
+          const error = message instanceof Error ? message : new Error(entry.message);
+          error[evaluatorErrorLogEntryKey] = entry;
+          return error;
+        };
         try {
-          const logs = [];
-          const logDiff = (expected, actual, options) => {
-            logs.push(createDiffLogEntry(expected, actual, options));
-          };
-          const log = (message, options) => {
-            logs.push(createLogEntry(message, options));
-          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
           const result = yield* Effect.promise(
-            () => Promise.resolve(
-              evaluateFn({
+            () => Promise.resolve().then(
+              () => evaluateFn({
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
@@ -686,10 +710,24 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                   datasetId: task.datasetId
                 },
                 logDiff,
-                log
+                log,
+                createError
               })
             )
           );
+          if (result instanceof Error) {
+            const evaluatorError = result;
+            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(result));
+            testCaseError = result.message;
+            evaluatorScores.push({
+              evaluatorId,
+              scores: [],
+              passed: false,
+              logs: logs.length > 0 ? logs : void 0
+            });
+            continue;
+          }
           const { scores, metrics } = normalizeResult(result);
           const passed2 = computeEvaluatorPassed(evaluator, result, scores);
           evaluatorScores.push({
@@ -700,11 +738,16 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
             logs: logs.length > 0 ? logs : void 0
           });
         } catch (error) {
+          if (error instanceof Error) {
+            const taggedEntry = error[evaluatorErrorLogEntryKey];
+            logs.push(taggedEntry ?? createLogEntry(error));
+          }
           testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
           evaluatorScores.push({
             evaluatorId,
             scores: [],
-            passed: false
+            passed: false,
+            logs: logs.length > 0 ? logs : void 0
           });
         }
       }
@@ -775,6 +818,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   );
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* Ref.make(0);
+  const startedRef = yield* Ref.make(0);
   const passedRef = yield* Ref.make(0);
   const failedRef = yield* Ref.make(0);
   const processTestCase = (testCaseItem) => processOneTestCase(
@@ -784,6 +828,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     publishEvent,
     persistenceQueue,
     updateSnapshot,
+    startedRef,
     completedRef,
     passedRef,
     failedRef
@@ -1435,7 +1480,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
 async function generateDatasetJsonCommandInk(runner, datasetName) {
   return new Promise((resolve5, reject) => {
     const app = render(
-      React2.createElement(GenerateView, {
+      React2__default.createElement(GenerateView, {
         runner,
         datasetName,
         onComplete: (err) => {
@@ -1592,7 +1637,9 @@ function RunView({
   );
   const [runInfo, setRunInfo] = useState(null);
   const [testCases, setTestCases] = useState([]);
+  const [startedEvaluations, setStartedEvaluations] = useState(0);
   const [completedEvaluations, setCompletedEvaluations] = useState(0);
+  const [runningEvaluations, setRunningEvaluations] = useState([]);
   const [summary, setSummary] = useState(null);
   const [evaluatorNameById, setEvaluatorNameById] = useState(/* @__PURE__ */ new Map());
   const runEval = useCallback(async () => {
@@ -1629,6 +1676,25 @@ function RunView({
     let overallScoreCount = 0;
     const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
+        if (event.type === "TestCaseStarted") {
+          setStartedEvaluations(event.startedTestCases);
+          setRunningEvaluations((prev) => {
+            const withoutDuplicate = prev.filter(
+              (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
+            );
+            return [
+              ...withoutDuplicate,
+              {
+                testCaseId: event.testCaseId,
+                name: event.testCaseName,
+                rerunIndex: event.rerunIndex,
+                rerunTotal: event.rerunTotal,
+                startedTestCases: event.startedTestCases,
+                totalTestCases: event.totalTestCases
+              }
+            ];
+          });
+        }
         if (event.type === "TestCaseProgress") {
           for (const item of event.evaluatorScores) {
             const numeric = toNumericScoreFromScores(item.scores);
@@ -1688,12 +1754,18 @@ function RunView({
               rerunTotal: event.rerunTotal,
               durationMs: events.reduce((s, e) => s + e.durationMs, 0),
               passed: events.every((e) => e.passed),
+              errorMessage: event.errorMessage,
               events,
               aggregatedEvaluatorScores,
               isAggregated
             };
             byId.set(event.testCaseId, merged);
             setCompletedEvaluations(event.completedTestCases);
+            setRunningEvaluations(
+              (running) => running.filter(
+                (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
+              )
+            );
             return Array.from(byId.values());
           });
         }
@@ -1769,12 +1841,30 @@ function RunView({
         runInfo.totalTestCases
       ] })
     ] }),
-    phase === "running" && /* @__PURE__ */ jsx(Box, { marginBottom: 1, children: /* @__PURE__ */ jsx(
-      Spinner,
-      {
-        label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0}`
-      }
-    ) }),
+    phase === "running" && /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 1, children: [
+      /* @__PURE__ */ jsx(
+        Spinner,
+        {
+          label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
+        }
+      ),
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxs(Text, { color: "yellow", children: [
+        "[running ",
+        item.startedTestCases,
+        "/",
+        item.totalTestCases,
+        "] ",
+        item.name,
+        " ",
+        /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+          "(",
+          item.rerunIndex,
+          "/",
+          item.rerunTotal,
+          ")"
+        ] })
+      ] }, `${item.testCaseId}:${item.rerunIndex}`)) })
+    ] }),
     testCases.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxs(Text, { children: [
         /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
@@ -1798,8 +1888,13 @@ function RunView({
           " (",
           tc.durationMs,
           "ms)"
-        ] })
+        ] }),
+        tc.errorMessage ? /* @__PURE__ */ jsxs(Text, { color: "red", bold: true, children: [
+          " ",
+          "ERROR"
+        ] }) : null
       ] }),
+      tc.errorMessage ? /* @__PURE__ */ jsx(Text, { color: "red", children: tc.errorMessage }) : null,
       tc.aggregatedEvaluatorScores.map((item) => /* @__PURE__ */ jsxs(
         Box,
         {
@@ -2244,9 +2339,11 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   let overallScoreTotal = 0;
   let overallScoreSumSq = 0;
   let overallScoreCount = 0;
+  let startedCount = 0;
   let completedCount = 0;
   let totalCount = 0;
   let runFinished = false;
+  const inFlightReruns = /* @__PURE__ */ new Set();
   const spinnerFrames = ["\u280B", "\u2819", "\u2838", "\u2834", "\u2826", "\u2807"];
   let spinnerIndex = 0;
   function clearLine() {
@@ -2270,7 +2367,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       `\r${colorize(frame, ansi2.cyan)} Running evaluations ${colorize(
         `${completedCount}/${totalCount}`,
         ansi2.bold
-      )} ${colorize("(live)", ansi2.dim)}`
+      )} completed ${colorize(`${startedCount}/${totalCount}`, ansi2.bold)} started ${colorize(`(${inFlightReruns.size} running)`, ansi2.dim)}`
     );
   }
   let lastPrintedTestCaseId = null;
@@ -2278,8 +2375,19 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   let spinnerTimer;
   const done = new Promise((resolve5) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
+      if (event.type === "TestCaseStarted") {
+        startedCount = event.startedTestCases;
+        inFlightReruns.add(`${event.testCaseId}:${event.rerunIndex}`);
+        clearLine();
+        process.stdout.write(
+          `${colorize(`[started ${event.startedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize("(running)", ansi2.dim)}
+`
+        );
+        drawSpinner();
+      }
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
+        inFlightReruns.delete(`${event.testCaseId}:${event.rerunIndex}`);
         const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
         const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, value) => sum + value, 0) / numericScores.length : void 0;
         const testCaseId = event.testCaseId;
@@ -2337,9 +2445,13 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
           0
         );
         const lines = [];
+        const statusSuffix = event.errorMessage ? ` ${colorize("ERROR", `${ansi2.bold}${ansi2.red}`)}` : "";
         lines.push(
-          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}`
+          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}${statusSuffix}`
         );
+        if (event.errorMessage) {
+          lines.push(colorize(event.errorMessage, ansi2.red));
+        }
         for (const item of aggregatedScores) {
           const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
           lines.push(