@m4trix/evals 0.24.0 → 0.25.1

package/dist/cli-simple.cjs

@@ -10,7 +10,7 @@ var promises = require('fs/promises');
 var url = require('url');
 var diff = require('diff');
 var stringify = require('fast-json-stable-stringify');
-var React2 = require('react');
+var React = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
 
@@ -37,25 +37,15 @@ function _interopNamespace(e) {
 
 var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
 var stringify__default = /*#__PURE__*/_interopDefault(stringify);
-var React2__namespace = /*#__PURE__*/_interopNamespace(React2);
+var React__namespace = /*#__PURE__*/_interopNamespace(React);
 
 // src/runner/config.ts
 var defaultRunnerConfig = {
   discovery: {
     rootDir: process.cwd(),
     datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
-    evaluatorSuffixes: [
-      ".evaluator.ts",
-      ".evaluator.tsx",
-      ".evaluator.js",
-      ".evaluator.mjs"
-    ],
-    testCaseSuffixes: [
-      ".test-case.ts",
-      ".test-case.tsx",
-      ".test-case.js",
-      ".test-case.mjs"
-    ],
+    evaluatorSuffixes: [".evaluator.ts", ".evaluator.tsx", ".evaluator.js", ".evaluator.mjs"],
+    testCaseSuffixes: [".test-case.ts", ".test-case.tsx", ".test-case.js", ".test-case.mjs"],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
   artifactDirectory: ".eval-results",
@@ -122,14 +112,15 @@ function getJitiLoader() {
   }
   const createJiti2 = jitiModule__namespace.createJiti ?? jitiModule__namespace.default;
   if (typeof createJiti2 !== "function") {
-    throw new Error(
-      "Failed to initialize jiti for m4trix eval config loading."
-    );
+    throw new Error("Failed to initialize jiti for m4trix eval config loading.");
   }
-  cachedLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
-    interopDefault: true,
-    moduleCache: true
-  });
+  cachedLoader = createJiti2(
+    (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)),
+    {
+      interopDefault: true,
+      moduleCache: true
+    }
+  );
   return cachedLoader;
 }
 function resolveConfigModuleExport(loadedModule) {
@@ -233,9 +224,7 @@ async function loadModuleExports(filePath) {
 }
 async function collectDatasetsFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
-  );
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.datasetSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -252,9 +241,7 @@ async function collectDatasetsFromFiles(config) {
 }
 async function collectEvaluatorsFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
-  );
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -271,9 +258,7 @@ async function collectEvaluatorsFromFiles(config) {
 }
 async function collectTestCasesFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
-  );
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.testCaseSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -345,16 +330,8 @@ function createDiffString(expected, actual, diffOptions) {
   const expectedProcessed = preprocessForDiff(expected, diffOptions);
   const actualProcessed = preprocessForDiff(actual, diffOptions);
   if (diffOptions?.keysOnly) {
-    const expectedKeys = JSON.stringify(
-      extractKeys(expectedProcessed),
-      null,
-      2
-    );
-    const actualKeys = JSON.stringify(
-      extractKeys(actualProcessed),
-      null,
-      2
-    );
+    const expectedKeys = JSON.stringify(extractKeys(expectedProcessed), null, 2);
+    const actualKeys = JSON.stringify(extractKeys(actualProcessed), null, 2);
     const parts2 = diff.diffLines(expectedKeys, actualKeys);
     return formatDiffParts(parts2);
   }
@@ -365,9 +342,7 @@ function createDiffString(expected, actual, diffOptions) {
   }
   const parts = diff.diffLines(expectedStr, actualStr);
   if (diffOptions?.outputNewOnly) {
-    const filtered = parts.filter(
-      (p) => p.added === true
-    );
+    const filtered = parts.filter((p) => p.added === true);
     return formatDiffParts(filtered);
   }
   return formatDiffParts(parts);
@@ -469,10 +444,7 @@ var ScoreAggregate = {
       const count = values.length || 1;
       const result = {};
       for (const field of fields) {
-        result[field] = values.reduce(
-          (s, v) => s + (v[field] ?? 0),
-          0
-        ) / count;
+        result[field] = values.reduce((s, v) => s + (v[field] ?? 0), 0) / count;
       }
       return result;
     };
@@ -506,13 +478,10 @@ var ScoreAggregate = {
           (s, v) => s + (v[valueField] ?? 0),
           0
         );
-        const sumSq = values.reduce(
-          (s, v) => {
-            const value = v[valueField] ?? 0;
-            return s + value * value;
-          },
-          0
-        );
+        const sumSq = values.reduce((s, v) => {
+          const value = v[valueField] ?? 0;
+          return s + value * value;
+        }, 0);
         const mean = sum / count;
         const variance = (sumSq - count * mean * mean) / (count - 1);
         stdDev = variance > 0 ? Math.sqrt(variance) : 0;
@@ -780,20 +749,14 @@ function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
 function createArtifactPath(artifactDirectory, datasetId, runId) {
-  return path.join(
-    artifactDirectory,
-    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
-  );
+  return path.join(artifactDirectory, `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`);
 }
 function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
   const { testCaseItem, rerunIndex, rerunTotal } = unit;
   return effect.Effect.gen(function* () {
     const evaluatorRunId = `run-${crypto.randomUUID()}`;
     const started = Date.now();
-    const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [
-      n + 1,
-      n + 1
-    ]);
+    const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [n + 1, n + 1]);
     yield* publishEvent({
       type: "TestCaseStarted",
       runId: task.runId,
@@ -826,9 +789,7 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
         return error;
       };
       try {
-        const ctx = yield* effect.Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
+        const ctx = yield* effect.Effect.promise(() => Promise.resolve(evaluator.resolveContext()));
         const result = yield* effect.Effect.promise(
           () => Promise.resolve().then(
             () => evaluateFn({
@@ -883,10 +844,7 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       }
     }
     const rerunPassedThis = evaluatorScores.every((s) => s.passed);
-    const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [
-      n + 1,
-      n + 1
-    ]);
+    const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [n + 1, n + 1]);
     const progressEvent = {
       type: "TestCaseProgress",
       runId: task.runId,
@@ -935,10 +893,7 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       } else {
         yield* effect.Ref.update(failedRef, (n) => n + 1);
       }
-      const [passed, failed] = yield* effect.Effect.all([
-        effect.Ref.get(passedRef),
-        effect.Ref.get(failedRef)
-      ]);
+      const [passed, failed] = yield* effect.Effect.all([effect.Ref.get(passedRef), effect.Ref.get(failedRef)]);
       yield* updateSnapshot(task.runId, (snapshot) => ({
         ...snapshot,
         passedTestCases: passed,
@@ -1258,15 +1213,11 @@ var EffectRunner = class {
     this.persistenceQueue = effect.Effect.runSync(
       effect.Queue.unbounded()
     );
-    this.snapshotsRef = effect.Effect.runSync(
-      effect.Ref.make(/* @__PURE__ */ new Map())
-    );
+    this.snapshotsRef = effect.Effect.runSync(effect.Ref.make(/* @__PURE__ */ new Map()));
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
-    this.schedulerFiber = effect.Effect.runFork(
-      this.createSchedulerEffect()
-    );
+    this.schedulerFiber = effect.Effect.runFork(this.createSchedulerEffect());
     this.persistenceFiber = effect.Effect.runFork(
       createPersistenceWorker(this.persistenceQueue)
     );
@@ -1413,9 +1364,9 @@ var EffectRunner = class {
     return effect.Effect.runSync(effect.Ref.get(this.snapshotsRef)).get(runId);
   }
   getAllRunSnapshots() {
-    return Array.from(
-      effect.Effect.runSync(effect.Ref.get(this.snapshotsRef)).values()
-    ).sort((a, b) => b.queuedAt - a.queuedAt);
+    return Array.from(effect.Effect.runSync(effect.Ref.get(this.snapshotsRef)).values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
   }
   async loadRunSnapshotsFromArtifacts() {
     return loadRunSnapshotsFromArtifacts(this.config);
@@ -1557,9 +1508,9 @@ function GenerateView({
   datasetName,
   onComplete
 }) {
-  const [result, setResult] = React2.useState(null);
-  const [error, setError] = React2.useState(null);
-  React2.useEffect(() => {
+  const [result, setResult] = React.useState(null);
+  const [error, setError] = React.useState(null);
+  React.useEffect(() => {
     let cancelled = false;
     async function run() {
       const dataset = await runner.resolveDatasetByName(datasetName);
@@ -1582,12 +1533,8 @@ function GenerateView({
       const absoluteDatasetPath = resolve5(process.cwd(), dataset.filePath);
       const parsed = parse2(absoluteDatasetPath);
       const outputPath = join4(parsed.dir, `${parsed.name}.cases.json`);
-      await writeFile2(
-        outputPath,
-        `${JSON.stringify(payload, null, 2)}
-`,
-        "utf8"
-      );
+      await writeFile2(outputPath, `${JSON.stringify(payload, null, 2)}
+`, "utf8");
       if (!cancelled) {
         setResult({
           count: payload.length,
@@ -1658,7 +1605,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
 async function generateDatasetJsonCommandInk(runner, datasetName) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
-      React2__namespace.default.createElement(GenerateView, {
+      React__namespace.default.createElement(GenerateView, {
         runner,
         datasetName,
         onComplete: (err) => {
@@ -1708,8 +1655,8 @@ function TextBar({
 }
 var FRAMES = ["\u280B", "\u2819", "\u2838", "\u2834", "\u2826", "\u2807"];
 function Spinner({ label = "Running" }) {
-  const [frame, setFrame] = React2.useState(0);
-  React2.useEffect(() => {
+  const [frame, setFrame] = React.useState(0);
+  React.useEffect(() => {
     const timer = setInterval(() => {
       setFrame((f) => (f + 1) % FRAMES.length);
     }, 100);
@@ -1743,9 +1690,7 @@ function createBar(value, max = 100, width = 20) {
 function aggregateEvaluatorScores(events, nameById) {
   if (events.length === 0)
     return [];
-  const evaluatorIds = new Set(
-    events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId))
-  );
+  const evaluatorIds = new Set(events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId)));
   const result = [];
   for (const evaluatorId of evaluatorIds) {
     const scoreIdToItems = /* @__PURE__ */ new Map();
@@ -1775,9 +1720,7 @@ function aggregateEvaluatorScores(events, nameById) {
       return es?.passed ?? false;
     });
     const lastEvent = events[events.length - 1];
-    const lastEs = lastEvent?.evaluatorScores.find(
-      (x) => x.evaluatorId === evaluatorId
-    );
+    const lastEs = lastEvent?.evaluatorScores.find((x) => x.evaluatorId === evaluatorId);
     result.push({
       evaluatorId,
       evaluatorName: nameById.get(evaluatorId) ?? evaluatorId,
@@ -1811,17 +1754,15 @@ function RunView({
   concurrency,
   onComplete
 }) {
-  const [phase, setPhase] = React2.useState(
-    "loading"
-  );
-  const [runInfo, setRunInfo] = React2.useState(null);
-  const [testCases, setTestCases] = React2.useState([]);
-  const [startedEvaluations, setStartedEvaluations] = React2.useState(0);
-  const [completedEvaluations, setCompletedEvaluations] = React2.useState(0);
-  const [runningEvaluations, setRunningEvaluations] = React2.useState([]);
-  const [summary, setSummary] = React2.useState(null);
-  const [evaluatorNameById, setEvaluatorNameById] = React2.useState(/* @__PURE__ */ new Map());
-  const runEval = React2.useCallback(async () => {
+  const [phase, setPhase] = React.useState("loading");
+  const [runInfo, setRunInfo] = React.useState(null);
+  const [testCases, setTestCases] = React.useState([]);
+  const [startedEvaluations, setStartedEvaluations] = React.useState(0);
+  const [completedEvaluations, setCompletedEvaluations] = React.useState(0);
+  const [runningEvaluations, setRunningEvaluations] = React.useState([]);
+  const [summary, setSummary] = React.useState(null);
+  const [evaluatorNameById, setEvaluatorNameById] = React.useState(/* @__PURE__ */ new Map());
+  const runEval = React.useCallback(async () => {
     const dataset = await runner.resolveDatasetByName(datasetName);
     if (!dataset) {
       const known = await runner.collectDatasets();
@@ -1920,10 +1861,7 @@ function RunView({
             };
             const events = existing ? [...existing.events, newEvent] : [newEvent];
             const isAggregated = events.length > 1;
-            const aggregatedEvaluatorScores = aggregateEvaluatorScores(
-              events,
-              nameById
-            );
+            const aggregatedEvaluatorScores = aggregateEvaluatorScores(events, nameById);
             const merged = {
               name: event.testCaseName,
               testCaseId: event.testCaseId,
@@ -1986,7 +1924,7 @@ function RunView({
     setPhase("completed");
     setTimeout(() => onComplete(), 200);
   }, [runner, datasetName, evaluatorPattern, concurrency, onComplete]);
-  React2.useEffect(() => {
+  React.useEffect(() => {
     void runEval();
   }, [runEval]);
   return /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", padding: 1, children: [
@@ -2028,30 +1966,22 @@ function RunView({
           label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
         }
       ),
-      runningEvaluations.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(
-        ink.Text,
-        {
-          color: "yellow",
-          children: [
-            "[running ",
-            item.startedTestCases,
-            "/",
-            item.totalTestCases,
-            "]",
-            " ",
-            item.name,
-            " ",
-            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-              "(",
-              item.rerunIndex,
-              "/",
-              item.rerunTotal,
-              ")"
-            ] })
-          ]
-        },
-        `${item.testCaseId}:${item.rerunIndex}`
-      )) })
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "yellow", children: [
+        "[running ",
+        item.startedTestCases,
+        "/",
+        item.totalTestCases,
+        "] ",
+        item.name,
+        " ",
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+          "(",
+          item.rerunIndex,
+          "/",
+          item.rerunTotal,
+          ")"
+        ] })
+      ] }, `${item.testCaseId}:${item.rerunIndex}`)) })
     ] }),
     testCases.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
@@ -2083,73 +2013,63 @@ function RunView({
         ] }) : null
       ] }),
       tc.errorMessage ? /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "red", children: tc.errorMessage }) : null,
-      tc.aggregatedEvaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(
-        ink.Box,
-        {
-          flexDirection: "column",
-          marginLeft: 2,
-          children: [
-            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-              item.evaluatorName,
-              ":",
-              " ",
-              /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-              item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+      tc.aggregatedEvaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginLeft: 2, children: [
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+          item.evaluatorName,
+          ":",
+          " ",
+          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
+          item.metrics && item.metrics.length > 0 ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
+            " ",
+            item.metrics.map((m) => {
+              const def = getMetricById(m.id);
+              if (!def)
+                return null;
+              const formatted = def.format(m.data, {
+                isAggregated: tc.isAggregated
+              });
+              const label = m.name ?? def.name;
+              return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+                "[",
+                label ? `${label}: ` : "",
+                formatted,
+                "]",
+                " "
+              ] }, m.id);
+            })
+          ] }) : null
+        ] }),
+        item.scores.length > 0 ? item.scores.map((s, idx) => {
+          const def = s.def ?? getScoreById(s.id);
+          const scoreLabel = s.name ?? def?.name ?? def?.id ?? s.id;
+          return /* @__PURE__ */ jsxRuntime.jsxs(
+            ink.Text,
+            {
+              color: scoreColor(toNumericScore(s.data) ?? 0),
+              children: [
+                "      ",
+                scoreLabel,
+                ":",
                 " ",
-                item.metrics.map((m) => {
-                  const def = getMetricById(m.id);
-                  if (!def)
-                    return null;
-                  const formatted = def.format(m.data, {
-                    isAggregated: tc.isAggregated
-                  });
-                  const label = m.name ?? def.name;
-                  return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-                    "[",
-                    label ? `${label}: ` : "",
-                    formatted,
-                    "]",
-                    " "
-                  ] }, m.id);
+                formatScorePart(s, scoreColor, {
+                  isAggregated: tc.isAggregated
                 })
-              ] }) : null
-            ] }),
-            item.scores.length > 0 ? item.scores.map((s, idx) => {
-              const def = s.def ?? getScoreById(s.id);
-              const scoreLabel = s.name ?? def?.name ?? def?.id ?? s.id;
-              return /* @__PURE__ */ jsxRuntime.jsxs(
-                ink.Text,
-                {
-                  color: scoreColor(toNumericScore(s.data) ?? 0),
-                  children: [
-                    "      ",
-                    scoreLabel,
-                    ":",
-                    " ",
-                    formatScorePart(s, scoreColor, {
-                      isAggregated: tc.isAggregated
-                    })
-                  ]
-                },
-                `${item.evaluatorId}-${s.id}-${idx}`
-              );
-            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: " n/a" }),
-            !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
-              (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(
-                ({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
-                  ink.Text,
-                  {
-                    color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
-                    children: line
-                  },
-                  lineIdx
-                )
-              ) }, logIdx) : log.type === "log" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getLogLines(log).map((line, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: line }, lineIdx)) }, logIdx) : null
-            ) })
-          ]
-        },
-        item.evaluatorId
-      ))
+              ]
+            },
+            `${item.evaluatorId}-${s.id}-${idx}`
+          );
+        }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: " n/a" }),
+        !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
+          (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
+            ink.Text,
+            {
+              color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+              children: line
+            },
+            lineIdx
+          )) }, logIdx) : log.type === "log" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getLogLines(log).map((line, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: line }, lineIdx)) }, logIdx) : null
+        ) })
+      ] }, item.evaluatorId))
     ] }, tc.testCaseId)) }),
     phase === "completed" && summary && /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
       /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Run Summary" }),
@@ -2191,9 +2111,9 @@ function RunView({
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          const scoreKeys = [
-            ...summary.scoreItemsByEvaluatorScore?.keys() ?? []
-          ].filter((k) => k.startsWith(`${id}:`));
+          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
+            (k) => k.startsWith(`${id}:`)
+          );
           if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
               "- ",
@@ -2223,19 +2143,12 @@ function RunView({
               const label = aggregated.name ?? def?.name ?? def?.id ?? aggregated.id;
               const formatted = def ? def.formatAggregate(aggregated.data) : "n/a";
               const numeric = toNumericScore(aggregated.data);
-              return /* @__PURE__ */ jsxRuntime.jsxs(
-                ink.Text,
-                {
-                  color: numeric !== void 0 ? scoreColor(numeric) : "gray",
-                  children: [
-                    "    ",
-                    label,
-                    ": ",
-                    formatted
-                  ]
-                },
-                key
-              );
+              return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: numeric !== void 0 ? scoreColor(numeric) : "gray", children: [
+                "    ",
+                label,
+                ": ",
+                formatted
+              ] }, key);
             })
           ] }, id);
         })
@@ -2311,9 +2224,7 @@ function buildTestCaseSummaries(byId) {
     for (const evaluatorScores of events[0]?.evaluatorScores ?? []) {
       const scoreIdToItems = /* @__PURE__ */ new Map();
       for (const ev of events) {
-        const es = ev.evaluatorScores.find(
-          (x) => x.evaluatorId === evaluatorScores.evaluatorId
-        );
+        const es = ev.evaluatorScores.find((x) => x.evaluatorId === evaluatorScores.evaluatorId);
         for (const s of es?.scores ?? []) {
           const list = scoreIdToItems.get(s.id) ?? [];
           list.push(s);
@@ -2366,9 +2277,7 @@ function scoreToColor(score) {
 }
 function getEvaluatorSummaryLines(evaluatorId, evaluatorName, aggregate, scoreItemsByKey) {
   const lines = [];
-  const scoreKeys = [...scoreItemsByKey.keys()].filter(
-    (k) => k.startsWith(`${evaluatorId}:`)
-  );
+  const scoreKeys = [...scoreItemsByKey.keys()].filter((k) => k.startsWith(`${evaluatorId}:`));
   if (scoreKeys.length === 0) {
     lines.push(`- ${evaluatorName.padEnd(28)} no scores`);
     return lines;
@@ -2403,9 +2312,7 @@ function createBar2(value, max = 100, width = 20) {
 function aggregateEvaluatorScoresFromEvents(events, _evaluatorNameById) {
   if (events.length === 0)
     return [];
-  const evaluatorIds = new Set(
-    events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId))
-  );
+  const evaluatorIds = new Set(events.flatMap((e) => e.evaluatorScores.map((x) => x.evaluatorId)));
   const result = [];
   for (const evaluatorId of evaluatorIds) {
     const scoreIdToItems = /* @__PURE__ */ new Map();
@@ -2452,9 +2359,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
       if (def) {
         const formatted = def.format(m.data, options);
         const label = m.name ?? def.name;
-        metricParts.push(
-          label ? `[${label}: ${formatted}]` : `[${formatted}]`
-        );
+        metricParts.push(label ? `[${label}: ${formatted}]` : `[${formatted}]`);
       }
     }
   }
@@ -2628,10 +2533,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
         const aggregatedScores = aggregateEvaluatorScoresFromEvents(
           existing.events);
         const isAggregated = existing.events.length > 1;
-        const durationMs = existing.events.reduce(
-          (s, e) => s + e.durationMs,
-          0
-        );
+        const durationMs = existing.events.reduce((s, e) => s + e.durationMs, 0);
         const lines = [];
         const statusSuffix = event.errorMessage ? ` ${colorize("ERROR", `${ansi2.bold}${ansi2.red}`)}` : "";
         lines.push(
@@ -2643,18 +2545,12 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
         for (const item of aggregatedScores) {
           const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
           lines.push(
-            ...formatEvaluatorScoreLine(
-              name,
-              item.scores,
-              item.passed,
-              item.metrics,
-              { isAggregated }
-            )
+            ...formatEvaluatorScoreLine(name, item.scores, item.passed, item.metrics, {
+              isAggregated
+            })
           );
           const lastEvent = existing.events[existing.events.length - 1];
-          const lastEs = lastEvent?.evaluatorScores.find(
-            (x) => x.evaluatorId === item.evaluatorId
-          );
+          const lastEs = lastEvent?.evaluatorScores.find((x) => x.evaluatorId === item.evaluatorId);
           if (!item.passed && lastEs?.logs && lastEs.logs.length > 0) {
             for (const log of lastEs.logs) {
               if (log.type === "diff") {
@@ -2701,9 +2597,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
   console.log(
     `Evaluators: ${evaluators.map((item) => item.evaluator.getName() ?? item.id).join(", ")}`
   );
-  console.log(
-    `Total test cases: ${colorize(String(snapshot.totalTestCases), ansi2.bold)}`
-  );
+  console.log(`Total test cases: ${colorize(String(snapshot.totalTestCases), ansi2.bold)}`);
   console.log("");
   drawSpinner();
   spinnerTimer = setInterval(drawSpinner, 100);
@@ -2718,10 +2612,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
   console.log("");
   console.log(colorize("=== Run Summary ===", `${ansi2.bold}${ansi2.cyan}`));
   console.log(
-    `- passed: ${colorize(
-      `${completed.passedTestCases}/${completed.totalTestCases}`,
-      ansi2.green
-    )}`
+    `- passed: ${colorize(`${completed.passedTestCases}/${completed.totalTestCases}`, ansi2.green)}`
   );
   console.log(
     `- failed: ${colorize(
@@ -2731,11 +2622,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
   );
   if (overallScoreCount > 0) {
     const overallAverage = overallScoreTotal / overallScoreCount;
-    const overallSd = sampleStdDev2(
-      overallScoreTotal,
-      overallScoreSumSq,
-      overallScoreCount
-    );
+    const overallSd = sampleStdDev2(overallScoreTotal, overallScoreSumSq, overallScoreCount);
     const avgStr = overallSd !== void 0 ? `${overallAverage.toFixed(2)} \xB1 ${overallSd.toFixed(2)}` : overallAverage.toFixed(2);
     console.log(
       `- overall avg score: ${colorize(
@@ -2784,7 +2671,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
 async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern, concurrency) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
-      React2__namespace.createElement(RunView, {
+      React__namespace.createElement(RunView, {
         runner,
         datasetName,
         evaluatorPattern,