@m4trix/evals 0.21.0 → 0.22.0

package/dist/cli-simple.js

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
+import { Effect, PubSub, Queue, Ref, Fiber } from 'effect';
 import { existsSync } from 'fs';
 import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffString } from 'json-diff';
+import { diffLines } from 'diff';
+import stringify from 'fast-json-stable-stringify';
+import { cpus } from 'os';
 import * as React2 from 'react';
 import React2__default, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
@@ -261,10 +263,102 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify(preprocessForDiff(a, options));
+      const bStr = stringify(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
+    }
+    return filtered;
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
+  }
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
+  }
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
+  }
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
+  }
+  return lines.join("\n");
+}
 function createDiffString(expected, actual, diffOptions) {
-  const opts = { ...diffOptions, color: false };
-  const result = diffString(expected, actual, opts);
-  return typeof result === "string" ? result : "";
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(
+      extractKeys(expectedProcessed),
+      null,
+      2
+    );
+    const actualKeys = JSON.stringify(
+      extractKeys(actualProcessed),
+      null,
+      2
+    );
+    const parts2 = diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
+  }
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
+  }
+  const parts = diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter(
+      (p) => p.added === true
+    );
+    return formatDiffParts(filtered);
+  }
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
+  }
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
+  }
+  return result;
 }
 function formatLogMessage(msg) {
   if (typeof msg === "string")
@@ -643,6 +737,20 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
+function buildEvaluationUnits(testCases) {
+  const units = [];
+  for (const testCaseItem of testCases) {
+    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    for (let r = 0; r < rerunTotal; r++) {
+      units.push({
+        testCaseItem,
+        rerunIndex: r + 1,
+        rerunTotal
+      });
+    }
+  }
+  return units;
+}
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -652,157 +760,171 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
+function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
+  const { testCaseItem, rerunIndex, rerunTotal } = unit;
   return Effect.gen(function* () {
-    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    const rerunPassed = [];
-    for (let r = 0; r < reruns; r++) {
-      const evaluatorRunId = `run-${randomUUID()}`;
-      const started = Date.now();
-      const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      yield* publishEvent({
-        type: "TestCaseStarted",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        startedTestCases: startedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns
-      });
-      const evaluatorScores = [];
-      let testCaseError;
-      const output = readOutput(testCaseItem.testCase);
-      for (const { id: evaluatorId, evaluator } of task.evaluators) {
-        const evaluateFn = evaluator.getEvaluateFn();
-        if (!evaluateFn) {
-          continue;
-        }
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const log = (message, options) => {
-          logs.push(createLogEntry(message, options));
-        };
-        const createError = (message, options) => {
-          const entry = createLogEntry(message, options);
-          const error = message instanceof Error ? message : new Error(entry.message);
-          error[evaluatorErrorLogEntryKey] = entry;
-          return error;
-        };
-        try {
-          const ctx = yield* Effect.promise(
-            () => Promise.resolve(evaluator.resolveContext())
-          );
-          const result = yield* Effect.promise(
-            () => Promise.resolve().then(
-              () => evaluateFn({
-                input: testCaseItem.testCase.getInput(),
-                ctx,
-                output,
-                meta: {
-                  triggerId: task.triggerId,
-                  runId: evaluatorRunId,
-                  datasetId: task.datasetId
-                },
-                logDiff,
-                log,
-                createError
-              })
-            )
-          );
-          if (result instanceof Error) {
-            const evaluatorError = result;
-            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(result));
-            testCaseError = result.message;
-            evaluatorScores.push({
-              evaluatorId,
-              scores: [],
-              passed: false,
-              logs: logs.length > 0 ? logs : void 0
-            });
-            continue;
-          }
-          const { scores, metrics } = normalizeResult(result);
-          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
-          evaluatorScores.push({
-            evaluatorId,
-            scores,
-            passed: passed2,
-            metrics,
-            logs: logs.length > 0 ? logs : void 0
-          });
-        } catch (error) {
-          if (error instanceof Error) {
-            const taggedEntry = error[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(error));
-          }
-          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+    const evaluatorRunId = `run-${randomUUID()}`;
+    const started = Date.now();
+    const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
+      n + 1,
+      n + 1
+    ]);
+    yield* publishEvent({
+      type: "TestCaseStarted",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      startedTestCases: startedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal
+    });
+    const evaluatorScores = [];
+    let testCaseError;
+    const output = readOutput(testCaseItem.testCase);
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      const logs = [];
+      const logDiff = (expected, actual, options) => {
+        logs.push(createDiffLogEntry(expected, actual, options));
+      };
+      const log = (message, options) => {
+        logs.push(createLogEntry(message, options));
+      };
+      const createError = (message, options) => {
+        const entry = createLogEntry(message, options);
+        const error = message instanceof Error ? message : new Error(entry.message);
+        error[evaluatorErrorLogEntryKey] = entry;
+        return error;
+      };
+      try {
+        const ctx = yield* Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* Effect.promise(
+          () => Promise.resolve().then(
+            () => evaluateFn({
+              input: testCaseItem.testCase.getInput(),
+              ctx,
+              output,
+              meta: {
+                triggerId: task.triggerId,
+                runId: evaluatorRunId,
+                datasetId: task.datasetId
+              },
+              logDiff,
+              log,
+              createError
+            })
+          )
+        );
+        if (result instanceof Error) {
+          const evaluatorError = result;
+          const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(result));
+          testCaseError = result.message;
           evaluatorScores.push({
             evaluatorId,
             scores: [],
             passed: false,
             logs: logs.length > 0 ? logs : void 0
           });
+          continue;
+        }
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
+      } catch (error) {
+        if (error instanceof Error) {
+          const taggedEntry = error[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(error));
         }
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false,
+          logs: logs.length > 0 ? logs : void 0
+        });
       }
-      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
-      rerunPassed.push(rerunPassedThis);
-      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      const progressEvent = {
-        type: "TestCaseProgress",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        completedTestCases: completedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns,
-        passed: rerunPassedThis,
-        durationMs: Date.now() - started,
-        evaluatorScores,
-        output,
-        errorMessage: testCaseError
-      };
-      updateSnapshot(task.runId, (snapshot) => ({
-        ...snapshot,
-        completedTestCases: completedEvaluations
-      }));
-      yield* publishEvent(progressEvent);
-      yield* Queue.offer(persistenceQueue, {
-        runId: task.runId,
-        artifactPath: task.snapshot.artifactPath,
-        payload: progressEvent
-      });
     }
-    const testCasePassed = rerunPassed.every(Boolean);
-    if (testCasePassed) {
-      yield* Ref.update(passedRef, (n) => n + 1);
-    } else {
-      yield* Ref.update(failedRef, (n) => n + 1);
-    }
-    const [passed, failed] = yield* Effect.all([
-      Ref.get(passedRef),
-      Ref.get(failedRef)
+    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+    const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+      n + 1,
+      n + 1
     ]);
-    updateSnapshot(task.runId, (snapshot) => ({
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases: completedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal,
+      passed: rerunPassedThis,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      output,
+      errorMessage: testCaseError
+    };
+    yield* updateSnapshot(task.runId, (snapshot) => ({
       ...snapshot,
-      passedTestCases: passed,
-      failedTestCases: failed
+      completedTestCases: completedEvaluations
     }));
+    yield* publishEvent(progressEvent);
+    yield* Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+    const testCaseCompleted = yield* Ref.modify(
+      testCaseResultsRef,
+      (map) => {
+        const key = testCaseItem.id;
+        const existing = map.get(key) ?? { completedCount: 0, results: [] };
+        const newResults = [...existing.results, rerunPassedThis];
+        const newCompletedCount = existing.completedCount + 1;
+        const isLast = newCompletedCount === rerunTotal;
+        const newMap = new Map(map);
+        newMap.set(key, {
+          completedCount: newCompletedCount,
+          results: newResults
+        });
+        const outcome = isLast ? newResults.every(Boolean) : null;
+        return [outcome, newMap];
+      }
+    );
+    if (testCaseCompleted !== null) {
+      if (testCaseCompleted) {
+        yield* Ref.update(passedRef, (n) => n + 1);
+      } else {
+        yield* Ref.update(failedRef, (n) => n + 1);
+      }
+      const [passed, failed] = yield* Effect.all([
+        Ref.get(passedRef),
+        Ref.get(failedRef)
+      ]);
+      yield* updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        passedTestCases: passed,
+        failedTestCases: failed
+      }));
+    }
   });
 }
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "running",
     startedAt
@@ -821,9 +943,13 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   const startedRef = yield* Ref.make(0);
   const passedRef = yield* Ref.make(0);
   const failedRef = yield* Ref.make(0);
-  const processTestCase = (testCaseItem) => processOneTestCase(
+  const testCaseResultsRef = yield* Ref.make(
+    /* @__PURE__ */ new Map()
+  );
+  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const processEvaluation = (unit) => processOneEvaluation(
     task,
-    testCaseItem,
+    unit,
     totalEvaluations,
     publishEvent,
     persistenceQueue,
@@ -831,11 +957,12 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     startedRef,
     completedRef,
     passedRef,
-    failedRef
+    failedRef,
+    testCaseResultsRef
   );
   yield* Effect.forEach(
-    task.testCases,
-    processTestCase,
+    evaluationUnits,
+    processEvaluation,
     maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
   );
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
@@ -853,7 +980,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
     completedTestCases: completedEvaluations,
@@ -1106,7 +1233,9 @@ var EffectRunner = class {
     this.persistenceQueue = Effect.runSync(
       Queue.unbounded()
     );
-    this.snapshots = /* @__PURE__ */ new Map();
+    this.snapshotsRef = Effect.runSync(
+      Ref.make(/* @__PURE__ */ new Map())
+    );
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
@@ -1209,7 +1338,13 @@ var EffectRunner = class {
       status: "queued",
       artifactPath
     };
-    this.snapshots.set(runId, snapshot);
+    await Effect.runPromise(
+      Ref.update(this.snapshotsRef, (map) => {
+        const next = new Map(map);
+        next.set(runId, snapshot);
+        return next;
+      })
+    );
     const queuedEvent = {
       type: "RunQueued",
       runId,
@@ -1250,12 +1385,12 @@ var EffectRunner = class {
     };
   }
   getRunSnapshot(runId) {
-    return this.snapshots.get(runId);
+    return Effect.runSync(Ref.get(this.snapshotsRef)).get(runId);
   }
   getAllRunSnapshots() {
-    return Array.from(this.snapshots.values()).sort(
-      (a, b) => b.queuedAt - a.queuedAt
-    );
+    return Array.from(
+      Effect.runSync(Ref.get(this.snapshotsRef)).values()
+    ).sort((a, b) => b.queuedAt - a.queuedAt);
   }
   async loadRunSnapshotsFromArtifacts() {
     return loadRunSnapshotsFromArtifacts(this.config);
@@ -1284,11 +1419,15 @@ var EffectRunner = class {
     );
   }
   updateSnapshot(runId, updater) {
-    const existing = this.snapshots.get(runId);
-    if (!existing) {
-      return;
-    }
-    this.snapshots.set(runId, updater(existing));
+    return Ref.modify(this.snapshotsRef, (map) => {
+      const existing = map.get(runId);
+      if (!existing) {
+        return [void 0, map];
+      }
+      const next = new Map(map);
+      next.set(runId, updater(existing));
+      return [void 0, next];
+    }).pipe(Effect.asVoid);
   }
   publishEvent(event) {
     return Effect.sync(() => {
@@ -1304,8 +1443,9 @@ var EffectRunner = class {
     );
   }
 };
-
-// src/cli-simple/args.ts
+function getDefaultConcurrency() {
+  return Math.max(1, cpus().length);
+}
 function parseSimpleCliArgs(argv) {
   const args = {
     help: false,
@@ -1332,6 +1472,14 @@ function parseSimpleCliArgs(argv) {
       index += 1;
       continue;
     }
+    if ((token === "--concurrency" || token === "-c") && argv[index + 1]) {
+      const n = parseInt(argv[index + 1], 10);
+      if (!Number.isNaN(n) && n >= 1) {
+        args.concurrency = n;
+      }
+      index += 1;
+      continue;
+    }
     args.unknownArgs.push(token);
   }
   return args;
@@ -1339,9 +1487,12 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>",
+    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern> [--concurrency N]",
     "  eval-agents-simple generate --dataset <datasetName>",
     "",
+    "Options:",
+    "  --concurrency, -c N   Max concurrent test cases (default: CPU count). Use 1 for sequential.",
+    "",
     "Pattern examples for --evaluator:",
     "  score-evaluator       exact name (case-insensitive)",
     '  "*score*"             wildcard pattern',
@@ -1630,6 +1781,7 @@ function RunView({
   runner,
   datasetName,
   evaluatorPattern,
+  concurrency,
   onComplete
 }) {
   const [phase, setPhase] = useState(
@@ -1777,7 +1929,8 @@ function RunView({
     });
     const snapshot = await runner.runDatasetWith({
       datasetId: dataset.id,
-      evaluatorIds: evaluators.map((item) => item.id)
+      evaluatorIds: evaluators.map((item) => item.id),
+      concurrency
     });
     setRunInfo({
       runId: snapshot.runId,
@@ -1805,7 +1958,7 @@ function RunView({
     });
     setPhase("completed");
     setTimeout(() => onComplete(), 200);
-  }, [runner, datasetName, evaluatorPattern, onComplete]);
+  }, [runner, datasetName, evaluatorPattern, concurrency, onComplete]);
   useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -1848,22 +2001,30 @@ function RunView({
           label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
         }
       ),
-      runningEvaluations.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxs(Text, { color: "yellow", children: [
-        "[running ",
-        item.startedTestCases,
-        "/",
-        item.totalTestCases,
-        "] ",
-        item.name,
-        " ",
-        /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-          "(",
-          item.rerunIndex,
-          "/",
-          item.rerunTotal,
-          ")"
-        ] })
-      ] }, `${item.testCaseId}:${item.rerunIndex}`)) })
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxs(
+        Text,
+        {
+          color: "yellow",
+          children: [
+            "[running ",
+            item.startedTestCases,
+            "/",
+            item.totalTestCases,
+            "]",
+            " ",
+            item.name,
+            " ",
+            /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+              "(",
+              item.rerunIndex,
+              "/",
+              item.rerunTotal,
+              ")"
+            ] })
+          ]
+        },
+        `${item.testCaseId}:${item.rerunIndex}`
+      )) })
     ] }),
     testCases.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxs(Text, { children: [
@@ -1945,7 +2106,7 @@ function RunView({
                 },
                 `${item.evaluatorId}-${s.id}-${idx}`
               );
-            }) : /* @__PURE__ */ jsx(Text, { color: "gray", children: "      n/a" }),
+            }) : /* @__PURE__ */ jsx(Text, { color: "gray", children: " n/a" }),
             !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsx(Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
               (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getDiffLines(log).map(
                 ({ type, line }, lineIdx) => /* @__PURE__ */ jsx(
@@ -2003,9 +2164,9 @@ function RunView({
         /* @__PURE__ */ jsx(Text, { color: "magenta", children: "evaluator averages" }),
         Array.from(evaluatorNameById.entries()).map(([id, name]) => {
           const agg = summary.aggregates.get(id);
-          const scoreKeys = [...summary.scoreItemsByEvaluatorScore?.keys() ?? []].filter(
-            (k) => k.startsWith(`${id}:`)
-          );
+          const scoreKeys = [
+            ...summary.scoreItemsByEvaluatorScore?.keys() ?? []
+          ].filter((k) => k.startsWith(`${id}:`));
           if (scoreKeys.length === 0) {
             return /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
               "- ",
@@ -2313,7 +2474,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern, concurrency) {
   const dataset = await runner.resolveDatasetByName(datasetName);
   if (!dataset) {
     const known = await runner.collectDatasets();
@@ -2503,7 +2664,8 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   });
   const snapshot = await runner.runDatasetWith({
     datasetId: dataset.id,
-    evaluatorIds: evaluators.map((item) => item.id)
+    evaluatorIds: evaluators.map((item) => item.id),
+    concurrency
   });
   totalCount = snapshot.totalTestCases;
   console.log(colorize("=== Eval Run Started ===", `${ansi2.bold}${ansi2.cyan}`));
@@ -2592,13 +2754,14 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   }
   console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
 }
-async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
+async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern, concurrency) {
   return new Promise((resolve5, reject) => {
     const app = render(
       React2.createElement(RunView, {
         runner,
         datasetName,
         evaluatorPattern,
+        concurrency,
         onComplete: (err) => {
           app.unmount();
           if (err) {
@@ -2645,10 +2808,12 @@ async function main() {
   const runner = createRunner();
   try {
     if (args.command === "run") {
+      const concurrency = args.concurrency ?? getDefaultConcurrency();
       await (useInk ? runSimpleEvalCommandInk : runSimpleEvalCommandPlain)(
         runner,
         args.datasetName,
-        args.evaluatorPattern
+        args.evaluatorPattern,
+        concurrency
       );
       return;
     }