@m4trix/evals 0.20.0 → 0.21.1

package/dist/cli-simple.cjs

@@ -8,7 +8,9 @@ var path = require('path');
 var jitiModule = require('jiti');
 var promises = require('fs/promises');
 var url = require('url');
-var jsonDiff = require('json-diff');
+var diff = require('diff');
+var stringify = require('fast-json-stable-stringify');
+var os = require('os');
 var React2 = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
@@ -35,7 +37,8 @@ function _interopNamespace(e) {
 }
 
 var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
-var React2__default = /*#__PURE__*/_interopDefault(React2);
+var stringify__default = /*#__PURE__*/_interopDefault(stringify);
+var React2__namespace = /*#__PURE__*/_interopNamespace(React2);
 
 // src/runner/config.ts
 var defaultRunnerConfig = {
@@ -286,10 +289,102 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify__default.default(preprocessForDiff(a, options));
+      const bStr = stringify__default.default(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
+    }
+    return filtered;
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
+  }
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
+  }
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify__default.default(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
+  }
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
+  }
+  return lines.join("\n");
+}
 function createDiffString(expected, actual, diffOptions) {
-  const opts = { ...diffOptions, color: false };
-  const result = jsonDiff.diffString(expected, actual, opts);
-  return typeof result === "string" ? result : "";
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(
+      extractKeys(expectedProcessed),
+      null,
+      2
+    );
+    const actualKeys = JSON.stringify(
+      extractKeys(actualProcessed),
+      null,
+      2
+    );
+    const parts2 = diff.diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
+  }
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
+  }
+  const parts = diff.diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter(
+      (p) => p.added === true
+    );
+    return formatDiffParts(filtered);
+  }
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
+  }
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
+  }
+  return result;
 }
 function formatLogMessage(msg) {
   if (typeof msg === "string")
@@ -677,13 +772,27 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
   return effect.Effect.gen(function* () {
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
       const evaluatorRunId = `run-${crypto.randomUUID()}`;
       const started = Date.now();
+      const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      yield* publishEvent({
+        type: "TestCaseStarted",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        startedTestCases: startedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns
+      });
       const evaluatorScores = [];
       let testCaseError;
       const output = readOutput(testCaseItem.testCase);
@@ -829,6 +938,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
   );
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* effect.Ref.make(0);
+  const startedRef = yield* effect.Ref.make(0);
   const passedRef = yield* effect.Ref.make(0);
   const failedRef = yield* effect.Ref.make(0);
   const processTestCase = (testCaseItem) => processOneTestCase(
@@ -838,6 +948,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     publishEvent,
     persistenceQueue,
     updateSnapshot,
+    startedRef,
     completedRef,
     passedRef,
     failedRef
@@ -1313,8 +1424,9 @@ var EffectRunner = class {
     );
   }
 };
-
-// src/cli-simple/args.ts
+function getDefaultConcurrency() {
+  return Math.max(1, os.cpus().length);
+}
 function parseSimpleCliArgs(argv) {
   const args = {
     help: false,
@@ -1341,6 +1453,14 @@ function parseSimpleCliArgs(argv) {
       index += 1;
       continue;
     }
+    if ((token === "--concurrency" || token === "-c") && argv[index + 1]) {
+      const n = parseInt(argv[index + 1], 10);
+      if (!Number.isNaN(n) && n >= 1) {
+        args.concurrency = n;
+      }
+      index += 1;
+      continue;
+    }
     args.unknownArgs.push(token);
   }
   return args;
@@ -1348,9 +1468,12 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>",
+    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern> [--concurrency N]",
     "  eval-agents-simple generate --dataset <datasetName>",
     "",
+    "Options:",
+    "  --concurrency, -c N   Max concurrent test cases (default: CPU count). Use 1 for sequential.",
+    "",
     "Pattern examples for --evaluator:",
     "  score-evaluator       exact name (case-insensitive)",
     '  "*score*"             wildcard pattern',
@@ -1489,7 +1612,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
 async function generateDatasetJsonCommandInk(runner, datasetName) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
-      React2__default.default.createElement(GenerateView, {
+      React2__namespace.default.createElement(GenerateView, {
         runner,
         datasetName,
         onComplete: (err) => {
@@ -1639,6 +1762,7 @@ function RunView({
   runner,
   datasetName,
   evaluatorPattern,
+  concurrency,
   onComplete
 }) {
   const [phase, setPhase] = React2.useState(
@@ -1646,7 +1770,9 @@ function RunView({
   );
   const [runInfo, setRunInfo] = React2.useState(null);
   const [testCases, setTestCases] = React2.useState([]);
+  const [startedEvaluations, setStartedEvaluations] = React2.useState(0);
   const [completedEvaluations, setCompletedEvaluations] = React2.useState(0);
+  const [runningEvaluations, setRunningEvaluations] = React2.useState([]);
   const [summary, setSummary] = React2.useState(null);
   const [evaluatorNameById, setEvaluatorNameById] = React2.useState(/* @__PURE__ */ new Map());
   const runEval = React2.useCallback(async () => {
@@ -1683,6 +1809,25 @@ function RunView({
     let overallScoreCount = 0;
     const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
+        if (event.type === "TestCaseStarted") {
+          setStartedEvaluations(event.startedTestCases);
+          setRunningEvaluations((prev) => {
+            const withoutDuplicate = prev.filter(
+              (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
+            );
+            return [
+              ...withoutDuplicate,
+              {
+                testCaseId: event.testCaseId,
+                name: event.testCaseName,
+                rerunIndex: event.rerunIndex,
+                rerunTotal: event.rerunTotal,
+                startedTestCases: event.startedTestCases,
+                totalTestCases: event.totalTestCases
+              }
+            ];
+          });
+        }
         if (event.type === "TestCaseProgress") {
           for (const item of event.evaluatorScores) {
             const numeric = toNumericScoreFromScores(item.scores);
@@ -1749,6 +1894,11 @@ function RunView({
             };
             byId.set(event.testCaseId, merged);
             setCompletedEvaluations(event.completedTestCases);
+            setRunningEvaluations(
+              (running) => running.filter(
+                (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
+              )
+            );
             return Array.from(byId.values());
           });
         }
@@ -1760,7 +1910,8 @@ function RunView({
     });
     const snapshot = await runner.runDatasetWith({
       datasetId: dataset.id,
-      evaluatorIds: evaluators.map((item) => item.id)
+      evaluatorIds: evaluators.map((item) => item.id),
+      concurrency
     });
     setRunInfo({
       runId: snapshot.runId,
@@ -1788,7 +1939,7 @@ function RunView({
     });
     setPhase("completed");
     setTimeout(() => onComplete(), 200);
-  }, [runner, datasetName, evaluatorPattern, onComplete]);
+  }, [runner, datasetName, evaluatorPattern, concurrency, onComplete]);
   React2.useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -1824,12 +1975,38 @@ function RunView({
         runInfo.totalTestCases
       ] })
     ] }),
-    phase === "running" && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginBottom: 1, children: /* @__PURE__ */ jsxRuntime.jsx(
-      Spinner,
-      {
-        label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0}`
-      }
-    ) }),
+    phase === "running" && /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 1, children: [
+      /* @__PURE__ */ jsxRuntime.jsx(
+        Spinner,
+        {
+          label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
+        }
+      ),
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(
+        ink.Text,
+        {
+          color: "yellow",
+          children: [
+            "[running ",
+            item.startedTestCases,
+            "/",
+            item.totalTestCases,
+            "]",
+            " ",
+            item.name,
+            " ",
+            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+              "(",
+              item.rerunIndex,
+              "/",
+              item.rerunTotal,
+              ")"
+            ] })
+          ]
+        },
+        `${item.testCaseId}:${item.rerunIndex}`
+      )) })
+    ] }),
     testCases.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
         /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "cyan", children: [
@@ -1910,7 +2087,7 @@ function RunView({
                 },
                 `${item.evaluatorId}-${s.id}-${idx}`
               );
-            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "      n/a" }),
+            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: " n/a" }),
             !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
               (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(
                 ({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
@@ -1968,9 +2145,9 @@ function RunView({
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
-            (k) => k.startsWith(`${id}:`)
-          );
+          const scoreKeys = [
+            ...summary.scoreItemsByEvaluatorScore?.keys() ?? []
+          ].filter((k) => k.startsWith(`${id}:`));
           if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
               "- ",
@@ -2278,7 +2455,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern, concurrency) {
   const dataset = await runner.resolveDatasetByName(datasetName);
   if (!dataset) {
     const known = await runner.collectDatasets();
@@ -2304,9 +2481,11 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   let overallScoreTotal = 0;
   let overallScoreSumSq = 0;
   let overallScoreCount = 0;
+  let startedCount = 0;
   let completedCount = 0;
   let totalCount = 0;
   let runFinished = false;
+  const inFlightReruns = /* @__PURE__ */ new Set();
   const spinnerFrames = ["\u280B", "\u2819", "\u2838", "\u2834", "\u2826", "\u2807"];
   let spinnerIndex = 0;
   function clearLine() {
@@ -2330,7 +2509,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
       `\r${colorize(frame, ansi2.cyan)} Running evaluations ${colorize(
         `${completedCount}/${totalCount}`,
         ansi2.bold
-      )} ${colorize("(live)", ansi2.dim)}`
+      )} completed ${colorize(`${startedCount}/${totalCount}`, ansi2.bold)} started ${colorize(`(${inFlightReruns.size} running)`, ansi2.dim)}`
     );
   }
   let lastPrintedTestCaseId = null;
@@ -2338,8 +2517,19 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   let spinnerTimer;
   const done = new Promise((resolve5) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
+      if (event.type === "TestCaseStarted") {
+        startedCount = event.startedTestCases;
+        inFlightReruns.add(`${event.testCaseId}:${event.rerunIndex}`);
+        clearLine();
+        process.stdout.write(
+          `${colorize(`[started ${event.startedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize("(running)", ansi2.dim)}
+`
+        );
+        drawSpinner();
+      }
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
+        inFlightReruns.delete(`${event.testCaseId}:${event.rerunIndex}`);
         const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
         const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, value) => sum + value, 0) / numericScores.length : void 0;
         const testCaseId = event.testCaseId;
@@ -2455,7 +2645,8 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   });
   const snapshot = await runner.runDatasetWith({
     datasetId: dataset.id,
-    evaluatorIds: evaluators.map((item) => item.id)
+    evaluatorIds: evaluators.map((item) => item.id),
+    concurrency
   });
   totalCount = snapshot.totalTestCases;
   console.log(colorize("=== Eval Run Started ===", `${ansi2.bold}${ansi2.cyan}`));
@@ -2544,13 +2735,14 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   }
   console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
 }
-async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern, concurrency) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
-      React2__default.default.createElement(RunView, {
+      React2__namespace.createElement(RunView, {
         runner,
         datasetName,
         evaluatorPattern,
+        concurrency,
         onComplete: (err) => {
           app.unmount();
           if (err) {
@@ -2597,10 +2789,12 @@ async function main() {
   const runner = createRunner();
   try {
     if (args.command === "run") {
+      const concurrency = args.concurrency ?? getDefaultConcurrency();
       await (useInk ? runSimpleEvalCommandInk : runSimpleEvalCommandPlain)(
         runner,
         args.datasetName,
-        args.evaluatorPattern
+        args.evaluatorPattern,
+        concurrency
       );
       return;
     }