@m4trix/evals 0.21.0 → 0.22.0

package/dist/cli-simple.cjs

@@ -8,12 +8,16 @@ var path = require('path');
 var jitiModule = require('jiti');
 var promises = require('fs/promises');
 var url = require('url');
-var jsonDiff = require('json-diff');
+var diff = require('diff');
+var stringify = require('fast-json-stable-stringify');
+var os = require('os');
 var React2 = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
 
 var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
 function _interopNamespace(e) {
   if (e && e.__esModule) return e;
   var n = Object.create(null);
@@ -33,6 +37,7 @@ function _interopNamespace(e) {
 }
 
 var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
+var stringify__default = /*#__PURE__*/_interopDefault(stringify);
 var React2__namespace = /*#__PURE__*/_interopNamespace(React2);
 
 // src/runner/config.ts
@@ -284,10 +289,102 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify__default.default(preprocessForDiff(a, options));
+      const bStr = stringify__default.default(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
+    }
+    return filtered;
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
+  }
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
+  }
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify__default.default(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
+  }
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
+  }
+  return lines.join("\n");
+}
 function createDiffString(expected, actual, diffOptions) {
-  const opts = { ...diffOptions, color: false };
-  const result = jsonDiff.diffString(expected, actual, opts);
-  return typeof result === "string" ? result : "";
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(
+      extractKeys(expectedProcessed),
+      null,
+      2
+    );
+    const actualKeys = JSON.stringify(
+      extractKeys(actualProcessed),
+      null,
+      2
+    );
+    const parts2 = diff.diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
+  }
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
+  }
+  const parts = diff.diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter(
+      (p) => p.added === true
+    );
+    return formatDiffParts(filtered);
+  }
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
+  }
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
+  }
+  return result;
 }
 function formatLogMessage(msg) {
   if (typeof msg === "string")
@@ -666,6 +763,20 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
+function buildEvaluationUnits(testCases) {
+  const units = [];
+  for (const testCaseItem of testCases) {
+    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    for (let r = 0; r < rerunTotal; r++) {
+      units.push({
+        testCaseItem,
+        rerunIndex: r + 1,
+        rerunTotal
+      });
+    }
+  }
+  return units;
+}
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -675,157 +786,171 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
+function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
+  const { testCaseItem, rerunIndex, rerunTotal } = unit;
   return effect.Effect.gen(function* () {
-    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    const rerunPassed = [];
-    for (let r = 0; r < reruns; r++) {
-      const evaluatorRunId = `run-${crypto.randomUUID()}`;
-      const started = Date.now();
-      const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      yield* publishEvent({
-        type: "TestCaseStarted",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        startedTestCases: startedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns
-      });
-      const evaluatorScores = [];
-      let testCaseError;
-      const output = readOutput(testCaseItem.testCase);
-      for (const { id: evaluatorId, evaluator } of task.evaluators) {
-        const evaluateFn = evaluator.getEvaluateFn();
-        if (!evaluateFn) {
-          continue;
-        }
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const log = (message, options) => {
-          logs.push(createLogEntry(message, options));
-        };
-        const createError = (message, options) => {
-          const entry = createLogEntry(message, options);
-          const error = message instanceof Error ? message : new Error(entry.message);
-          error[evaluatorErrorLogEntryKey] = entry;
-          return error;
-        };
-        try {
-          const ctx = yield* effect.Effect.promise(
-            () => Promise.resolve(evaluator.resolveContext())
-          );
-          const result = yield* effect.Effect.promise(
-            () => Promise.resolve().then(
-              () => evaluateFn({
-                input: testCaseItem.testCase.getInput(),
-                ctx,
-                output,
-                meta: {
-                  triggerId: task.triggerId,
-                  runId: evaluatorRunId,
-                  datasetId: task.datasetId
-                },
-                logDiff,
-                log,
-                createError
-              })
-            )
-          );
-          if (result instanceof Error) {
-            const evaluatorError = result;
-            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(result));
-            testCaseError = result.message;
-            evaluatorScores.push({
-              evaluatorId,
-              scores: [],
-              passed: false,
-              logs: logs.length > 0 ? logs : void 0
-            });
-            continue;
-          }
-          const { scores, metrics } = normalizeResult(result);
-          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
-          evaluatorScores.push({
-            evaluatorId,
-            scores,
-            passed: passed2,
-            metrics,
-            logs: logs.length > 0 ? logs : void 0
-          });
-        } catch (error) {
-          if (error instanceof Error) {
-            const taggedEntry = error[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(error));
-          }
-          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+    const evaluatorRunId = `run-${crypto.randomUUID()}`;
+    const started = Date.now();
+    const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [
+      n + 1,
+      n + 1
+    ]);
+    yield* publishEvent({
+      type: "TestCaseStarted",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      startedTestCases: startedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal
+    });
+    const evaluatorScores = [];
+    let testCaseError;
+    const output = readOutput(testCaseItem.testCase);
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      const logs = [];
+      const logDiff = (expected, actual, options) => {
+        logs.push(createDiffLogEntry(expected, actual, options));
+      };
+      const log = (message, options) => {
+        logs.push(createLogEntry(message, options));
+      };
+      const createError = (message, options) => {
+        const entry = createLogEntry(message, options);
+        const error = message instanceof Error ? message : new Error(entry.message);
+        error[evaluatorErrorLogEntryKey] = entry;
+        return error;
+      };
+      try {
+        const ctx = yield* effect.Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* effect.Effect.promise(
+          () => Promise.resolve().then(
+            () => evaluateFn({
+              input: testCaseItem.testCase.getInput(),
+              ctx,
+              output,
+              meta: {
+                triggerId: task.triggerId,
+                runId: evaluatorRunId,
+                datasetId: task.datasetId
+              },
+              logDiff,
+              log,
+              createError
+            })
+          )
+        );
+        if (result instanceof Error) {
+          const evaluatorError = result;
+          const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(result));
+          testCaseError = result.message;
           evaluatorScores.push({
             evaluatorId,
             scores: [],
             passed: false,
             logs: logs.length > 0 ? logs : void 0
           });
+          continue;
+        }
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
+      } catch (error) {
+        if (error instanceof Error) {
+          const taggedEntry = error[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(error));
         }
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false,
+          logs: logs.length > 0 ? logs : void 0
+        });
       }
-      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
-      rerunPassed.push(rerunPassedThis);
-      const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      const progressEvent = {
-        type: "TestCaseProgress",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        completedTestCases: completedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns,
-        passed: rerunPassedThis,
-        durationMs: Date.now() - started,
-        evaluatorScores,
-        output,
-        errorMessage: testCaseError
-      };
-      updateSnapshot(task.runId, (snapshot) => ({
-        ...snapshot,
-        completedTestCases: completedEvaluations
-      }));
-      yield* publishEvent(progressEvent);
-      yield* effect.Queue.offer(persistenceQueue, {
-        runId: task.runId,
-        artifactPath: task.snapshot.artifactPath,
-        payload: progressEvent
-      });
     }
-    const testCasePassed = rerunPassed.every(Boolean);
-    if (testCasePassed) {
-      yield* effect.Ref.update(passedRef, (n) => n + 1);
-    } else {
-      yield* effect.Ref.update(failedRef, (n) => n + 1);
-    }
-    const [passed, failed] = yield* effect.Effect.all([
-      effect.Ref.get(passedRef),
-      effect.Ref.get(failedRef)
+    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+    const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [
+      n + 1,
+      n + 1
     ]);
-    updateSnapshot(task.runId, (snapshot) => ({
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases: completedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal,
+      passed: rerunPassedThis,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      output,
+      errorMessage: testCaseError
+    };
+    yield* updateSnapshot(task.runId, (snapshot) => ({
       ...snapshot,
-      passedTestCases: passed,
-      failedTestCases: failed
+      completedTestCases: completedEvaluations
     }));
+    yield* publishEvent(progressEvent);
+    yield* effect.Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+    const testCaseCompleted = yield* effect.Ref.modify(
+      testCaseResultsRef,
+      (map) => {
+        const key = testCaseItem.id;
+        const existing = map.get(key) ?? { completedCount: 0, results: [] };
+        const newResults = [...existing.results, rerunPassedThis];
+        const newCompletedCount = existing.completedCount + 1;
+        const isLast = newCompletedCount === rerunTotal;
+        const newMap = new Map(map);
+        newMap.set(key, {
+          completedCount: newCompletedCount,
+          results: newResults
+        });
+        const outcome = isLast ? newResults.every(Boolean) : null;
+        return [outcome, newMap];
+      }
+    );
+    if (testCaseCompleted !== null) {
+      if (testCaseCompleted) {
+        yield* effect.Ref.update(passedRef, (n) => n + 1);
+      } else {
+        yield* effect.Ref.update(failedRef, (n) => n + 1);
+      }
+      const [passed, failed] = yield* effect.Effect.all([
+        effect.Ref.get(passedRef),
+        effect.Ref.get(failedRef)
+      ]);
+      yield* updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        passedTestCases: passed,
+        failedTestCases: failed
+      }));
+    }
   });
 }
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => effect.Effect.gen(function* () {
   const startedAt = Date.now();
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "running",
     startedAt
@@ -844,9 +969,13 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
   const startedRef = yield* effect.Ref.make(0);
   const passedRef = yield* effect.Ref.make(0);
   const failedRef = yield* effect.Ref.make(0);
-  const processTestCase = (testCaseItem) => processOneTestCase(
+  const testCaseResultsRef = yield* effect.Ref.make(
+    /* @__PURE__ */ new Map()
+  );
+  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const processEvaluation = (unit) => processOneEvaluation(
     task,
-    testCaseItem,
+    unit,
     totalEvaluations,
     publishEvent,
     persistenceQueue,
@@ -854,11 +983,12 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     startedRef,
     completedRef,
     passedRef,
-    failedRef
+    failedRef,
+    testCaseResultsRef
   );
   yield* effect.Effect.forEach(
-    task.testCases,
-    processTestCase,
+    evaluationUnits,
+    processEvaluation,
     maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
   );
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* effect.Effect.all([
@@ -876,7 +1006,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
     completedTestCases: completedEvaluations,
@@ -1129,7 +1259,9 @@ var EffectRunner = class {
     this.persistenceQueue = effect.Effect.runSync(
       effect.Queue.unbounded()
     );
-    this.snapshots = /* @__PURE__ */ new Map();
+    this.snapshotsRef = effect.Effect.runSync(
+      effect.Ref.make(/* @__PURE__ */ new Map())
+    );
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
@@ -1232,7 +1364,13 @@ var EffectRunner = class {
       status: "queued",
       artifactPath
     };
-    this.snapshots.set(runId, snapshot);
+    await effect.Effect.runPromise(
+      effect.Ref.update(this.snapshotsRef, (map) => {
+        const next = new Map(map);
+        next.set(runId, snapshot);
+        return next;
+      })
+    );
     const queuedEvent = {
       type: "RunQueued",
       runId,
@@ -1273,12 +1411,12 @@ var EffectRunner = class {
     };
   }
   getRunSnapshot(runId) {
-    return this.snapshots.get(runId);
+    return effect.Effect.runSync(effect.Ref.get(this.snapshotsRef)).get(runId);
   }
   getAllRunSnapshots() {
-    return Array.from(this.snapshots.values()).sort(
-      (a, b) => b.queuedAt - a.queuedAt
-    );
+    return Array.from(
+      effect.Effect.runSync(effect.Ref.get(this.snapshotsRef)).values()
+    ).sort((a, b) => b.queuedAt - a.queuedAt);
   }
   async loadRunSnapshotsFromArtifacts() {
     return loadRunSnapshotsFromArtifacts(this.config);
@@ -1307,11 +1445,15 @@ var EffectRunner = class {
     );
   }
   updateSnapshot(runId, updater) {
-    const existing = this.snapshots.get(runId);
-    if (!existing) {
-      return;
-    }
-    this.snapshots.set(runId, updater(existing));
+    return effect.Ref.modify(this.snapshotsRef, (map) => {
+      const existing = map.get(runId);
+      if (!existing) {
+        return [void 0, map];
+      }
+      const next = new Map(map);
+      next.set(runId, updater(existing));
+      return [void 0, next];
+    }).pipe(effect.Effect.asVoid);
   }
   publishEvent(event) {
     return effect.Effect.sync(() => {
@@ -1327,8 +1469,9 @@ var EffectRunner = class {
     );
   }
 };
-
-// src/cli-simple/args.ts
+function getDefaultConcurrency() {
+  return Math.max(1, os.cpus().length);
+}
 function parseSimpleCliArgs(argv) {
   const args = {
     help: false,
@@ -1355,6 +1498,14 @@ function parseSimpleCliArgs(argv) {
       index += 1;
       continue;
     }
+    if ((token === "--concurrency" || token === "-c") && argv[index + 1]) {
+      const n = parseInt(argv[index + 1], 10);
+      if (!Number.isNaN(n) && n >= 1) {
+        args.concurrency = n;
+      }
+      index += 1;
+      continue;
+    }
     args.unknownArgs.push(token);
   }
   return args;
@@ -1362,9 +1513,12 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>",
+    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern> [--concurrency N]",
     "  eval-agents-simple generate --dataset <datasetName>",
     "",
+    "Options:",
+    "  --concurrency, -c N   Max concurrent test cases (default: CPU count). Use 1 for sequential.",
+    "",
     "Pattern examples for --evaluator:",
     "  score-evaluator       exact name (case-insensitive)",
     '  "*score*"             wildcard pattern',
@@ -1653,6 +1807,7 @@ function RunView({
   runner,
   datasetName,
   evaluatorPattern,
+  concurrency,
   onComplete
 }) {
   const [phase, setPhase] = React2.useState(
@@ -1800,7 +1955,8 @@ function RunView({
     });
     const snapshot = await runner.runDatasetWith({
       datasetId: dataset.id,
-      evaluatorIds: evaluators.map((item) => item.id)
+      evaluatorIds: evaluators.map((item) => item.id),
+      concurrency
     });
     setRunInfo({
       runId: snapshot.runId,
@@ -1828,7 +1984,7 @@ function RunView({
     });
     setPhase("completed");
     setTimeout(() => onComplete(), 200);
-  }, [runner, datasetName, evaluatorPattern, onComplete]);
+  }, [runner, datasetName, evaluatorPattern, concurrency, onComplete]);
   React2.useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -1871,22 +2027,30 @@ function RunView({
           label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
         }
       ),
-      runningEvaluations.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "yellow", children: [
-        "[running ",
-        item.startedTestCases,
-        "/",
-        item.totalTestCases,
-        "] ",
-        item.name,
-        " ",
-        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-          "(",
-          item.rerunIndex,
-          "/",
-          item.rerunTotal,
-          ")"
-        ] })
-      ] }, `${item.testCaseId}:${item.rerunIndex}`)) })
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(
+        ink.Text,
+        {
+          color: "yellow",
+          children: [
+            "[running ",
+            item.startedTestCases,
+            "/",
+            item.totalTestCases,
+            "]",
+            " ",
+            item.name,
+            " ",
+            /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+              "(",
+              item.rerunIndex,
+              "/",
+              item.rerunTotal,
+              ")"
+            ] })
+          ]
+        },
+        `${item.testCaseId}:${item.rerunIndex}`
+      )) })
     ] }),
     testCases.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
@@ -1968,7 +2132,7 @@ function RunView({
                 },
                 `${item.evaluatorId}-${s.id}-${idx}`
               );
-            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: "      n/a" }),
+            }) : /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "gray", children: " n/a" }),
             !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
               (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(
                 ({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
@@ -2026,9 +2190,9 @@ function RunView({
         /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
-            (k) => k.startsWith(`${id}:`)
-          );
+          const scoreKeys = [
+            ...summary.scoreItemsByEvaluatorScore?.keys() ?? []
+          ].filter((k) => k.startsWith(`${id}:`));
           if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
               "- ",
@@ -2336,7 +2500,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern, concurrency) {
   const dataset = await runner.resolveDatasetByName(datasetName);
   if (!dataset) {
     const known = await runner.collectDatasets();
@@ -2526,7 +2690,8 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   });
   const snapshot = await runner.runDatasetWith({
     datasetId: dataset.id,
-    evaluatorIds: evaluators.map((item) => item.id)
+    evaluatorIds: evaluators.map((item) => item.id),
+    concurrency
   });
   totalCount = snapshot.totalTestCases;
   console.log(colorize("=== Eval Run Started ===", `${ansi2.bold}${ansi2.cyan}`));
@@ -2615,13 +2780,14 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   }
   console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
 }
-async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern, concurrency) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__namespace.createElement(RunView, {
         runner,
         datasetName,
         evaluatorPattern,
+        concurrency,
         onComplete: (err) => {
           app.unmount();
           if (err) {
@@ -2668,10 +2834,12 @@ async function main() {
   const runner = createRunner();
   try {
     if (args.command === "run") {
+      const concurrency = args.concurrency ?? getDefaultConcurrency();
       await (useInk ? runSimpleEvalCommandInk : runSimpleEvalCommandPlain)(
         runner,
         args.datasetName,
-        args.evaluatorPattern
+        args.evaluatorPattern,
+        concurrency
       );
       return;
     }