@m4trix/evals 0.25.1 → 0.27.0

package/dist/cli-simple.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Ref, Fiber } from 'effect';
+import { Schema, Effect, PubSub, Queue, Ref, Fiber, Either, ParseResult } from 'effect';
+import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
+import { resolve, join, relative, parse, dirname } from 'path';
 import { existsSync } from 'fs';
-import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
-import { writeFile, readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
 import { diffLines } from 'diff';
 import stringify from 'fast-json-stable-stringify';
@@ -13,12 +13,179 @@ import React__default, { useState, useEffect, useCallback } from 'react';
 import { render, Box, Text } from 'ink';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
 
+var ENTITY_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
+function makeEntityIdSchema(brand, label) {
+  return Schema.String.pipe(
+    Schema.trimmed(),
+    Schema.minLength(1, {
+      message: () => `${label} must be non-empty.`
+    }),
+    Schema.pattern(ENTITY_ID_PATTERN, {
+      message: () => `${label} may only contain letters, digits, underscores, and hyphens (no spaces). Examples: "my-nightly", "my_nightly", "myNightly".`
+    }),
+    Schema.brand(brand)
+  );
+}
+var RunConfigNameSchema = makeEntityIdSchema("RunConfigName", "RunConfig name");
+makeEntityIdSchema("EvaluatorName", "Evaluator name");
+makeEntityIdSchema("TestCaseName", "Test case name");
+makeEntityIdSchema("DatasetName", "Dataset name");
+function validateWithSchema(schema, raw, context) {
+  const trimmed = raw.trim();
+  const decode = Schema.decodeUnknownEither(
+    schema
+  );
+  const result = decode(trimmed);
+  if (Either.isLeft(result)) {
+    throw new Error(`${context}: ${ParseResult.TreeFormatter.formatErrorSync(result.left)}`);
+  }
+  return result.right;
+}
+function validateRunConfigName(raw, context) {
+  return validateWithSchema(RunConfigNameSchema, raw, context);
+}
+
+// src/evals/evaluator.ts
+function getEvaluatorDisplayLabel(evaluator) {
+  if (typeof evaluator.getDisplayLabel === "function") {
+    const label = evaluator.getDisplayLabel();
+    if (label !== void 0) {
+      return label;
+    }
+  }
+  return typeof evaluator.getName === "function" ? evaluator.getName() : void 0;
+}
+function getEvaluatorTagList(evaluator) {
+  return typeof evaluator.getTags === "function" ? [...evaluator.getTags()] : [];
+}
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
+      }
+    } catch {
+    }
+  }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
+
 // src/runner/config.ts
 var defaultRunnerConfig = {
   discovery: {
     rootDir: process.cwd(),
     datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
     evaluatorSuffixes: [".evaluator.ts", ".evaluator.tsx", ".evaluator.js", ".evaluator.mjs"],
+    runConfigSuffixes: [".run-config.ts", ".run-config.tsx", ".run-config.js", ".run-config.mjs"],
     testCaseSuffixes: [".test-case.ts", ".test-case.tsx", ".test-case.js", ".test-case.mjs"],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
@@ -44,6 +211,11 @@ function toRunnerConfigOverrides(config) {
   } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
     discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
   }
+  if (rawDiscovery?.runConfigFilePatterns !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigFilePatterns;
+  } else if (rawDiscovery?.runConfigSuffixes !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigSuffixes;
+  }
   if (rawDiscovery?.testCaseFilePatterns !== void 0) {
     discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
   } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
@@ -142,6 +314,9 @@ function isDatasetLike(value) {
 function isEvaluatorLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
 }
+function isRunConfigLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getRuns") && typeof value.getRuns === "function";
+}
 function isTestCaseLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
 }
@@ -230,6 +405,23 @@ async function collectEvaluatorsFromFiles(config) {
   );
   return found.flat();
 }
+async function collectRunConfigsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.runConfigSuffixes));
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const runConfigs = exports.filter(isRunConfigLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return runConfigs.map((runConfig) => ({
+        id: runConfig.getName(),
+        filePath: relPath,
+        runConfig
+      }));
+    })
+  );
+  return found.flat();
+}
 async function collectTestCasesFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
   const matched = files.filter((filePath) => hasOneSuffix(filePath, config.testCaseSuffixes));
@@ -383,6 +575,25 @@ function getDiffLines(entry) {
   });
 }
 
+// src/evals/test-case.ts
+function getTestCaseDisplayLabel(testCase) {
+  if (typeof testCase.getDisplayLabel === "function") {
+    return testCase.getDisplayLabel();
+  }
+  return typeof testCase.getName === "function" ? testCase.getName() : "";
+}
+function getTestCaseTagList(testCase) {
+  return typeof testCase.getTags === "function" ? [...testCase.getTags()] : [];
+}
+
+// src/evals/dataset.ts
+function getDatasetDisplayLabel(dataset) {
+  if (typeof dataset.getDisplayLabel === "function") {
+    return dataset.getDisplayLabel();
+  }
+  return typeof dataset.getName === "function" ? dataset.getName() : "";
+}
+
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
 var Metric = {
@@ -406,6 +617,54 @@ function getMetricById(id) {
   return registry.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
 // src/evals/score.ts
 var registry2 = /* @__PURE__ */ new Map();
 function formatScoreData(def, data, options) {
@@ -514,54 +773,6 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
-// src/evals/aggregators.ts
-function aggregateTokenCountSum(values) {
-  const initial = {
-    input: 0,
-    output: 0,
-    inputCached: 0,
-    outputCached: 0
-  };
-  return values.reduce(
-    (acc, v) => ({
-      input: acc.input + (v.input ?? 0),
-      output: acc.output + (v.output ?? 0),
-      inputCached: acc.inputCached + (v.inputCached ?? 0),
-      outputCached: acc.outputCached + (v.outputCached ?? 0)
-    }),
-    initial
-  );
-}
-function aggregateLatencyAverage(values) {
-  if (values.length === 0) {
-    return { ms: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.ms, 0);
-  return { ms: sum / values.length };
-}
-
-// src/evals/metrics/standard.ts
-Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  aggregate: aggregateTokenCountSum,
-  format: (data, options) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    const base = `in:${input} out:${output} cached:${cached}`;
-    return options?.isAggregated ? `Total: ${base}` : base;
-  }
-});
-Metric.of({
-  id: "latency",
-  name: "Latency",
-  aggregate: aggregateLatencyAverage,
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
-});
-
 // src/evals/scores/standard.ts
 Score.of({
   id: "percent",
@@ -705,15 +916,17 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
-function buildEvaluationUnits(testCases) {
+function buildEvaluationUnits(testCases, repetitionCount) {
+  const count = Math.max(1, repetitionCount);
   const units = [];
   for (const testCaseItem of testCases) {
-    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    for (let r = 0; r < rerunTotal; r++) {
+    const repetitionId = `rep-${randomUUID()}`;
+    for (let r = 0; r < count; r++) {
       units.push({
         testCaseItem,
-        rerunIndex: r + 1,
-        rerunTotal
+        repetitionId,
+        repetitionIndex: r + 1,
+        repetitionCount: count
       });
     }
   }
@@ -726,7 +939,7 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
   return join(artifactDirectory, `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`);
 }
 function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
-  const { testCaseItem, rerunIndex, rerunTotal } = unit;
+  const { testCaseItem, repetitionId, repetitionIndex, repetitionCount } = unit;
   return Effect.gen(function* () {
     const evaluatorRunId = `run-${randomUUID()}`;
     const started = Date.now();
@@ -735,11 +948,12 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       type: "TestCaseStarted",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       startedTestCases: startedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal
+      repetitionId,
+      repetitionIndex,
+      repetitionCount
     });
     const evaluatorScores = [];
     let testCaseError;
@@ -773,8 +987,15 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
               meta: {
                 triggerId: task.triggerId,
                 runId: evaluatorRunId,
-                datasetId: task.datasetId
+                datasetName: task.dataset.getDisplayLabel(),
+                repetitionId,
+                repetitionIndex,
+                repetitionCount,
+                runConfigName: task.runConfigName
               },
+              testCaseTags: getTestCaseTagList(testCaseItem.testCase),
+              runConfigTags: task.runConfigTags,
+              evaluatorTags: getEvaluatorTagList(evaluator),
               logDiff,
               log,
               createError
@@ -817,18 +1038,19 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
         });
       }
     }
-    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+    const repetitionPassedThis = evaluatorScores.every((s) => s.passed);
     const completedEvaluations = yield* Ref.modify(completedRef, (n) => [n + 1, n + 1]);
     const progressEvent = {
       type: "TestCaseProgress",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       completedTestCases: completedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal,
-      passed: rerunPassedThis,
+      repetitionId,
+      repetitionIndex,
+      repetitionCount,
+      passed: repetitionPassedThis,
       durationMs: Date.now() - started,
       evaluatorScores,
       output,
@@ -849,9 +1071,9 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       (map) => {
         const key = testCaseItem.id;
         const existing = map.get(key) ?? { completedCount: 0, results: [] };
-        const newResults = [...existing.results, rerunPassedThis];
+        const newResults = [...existing.results, repetitionPassedThis];
         const newCompletedCount = existing.completedCount + 1;
-        const isLast = newCompletedCount === rerunTotal;
+        const isLast = newCompletedCount === repetitionCount;
         const newMap = new Map(map);
         newMap.set(key, {
           completedCount: newCompletedCount,
@@ -888,10 +1110,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  const totalEvaluations = task.testCases.reduce(
-    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-    0
-  );
+  const totalEvaluations = task.testCases.length * Math.max(1, task.repetitions);
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* Ref.make(0);
   const startedRef = yield* Ref.make(0);
@@ -900,7 +1119,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   const testCaseResultsRef = yield* Ref.make(
     /* @__PURE__ */ new Map()
   );
-  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const evaluationUnits = buildEvaluationUnits(task.testCases, task.repetitions);
   const processEvaluation = (unit) => processOneEvaluation(
     task,
     unit,
@@ -914,11 +1133,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     failedRef,
     testCaseResultsRef
   );
-  yield* Effect.forEach(
-    evaluationUnits,
-    processEvaluation,
-    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
-  );
+  const globalSem = task.globalEvaluationSemaphore;
+  if (globalSem !== void 0) {
+    yield* Effect.forEach(
+      evaluationUnits,
+      (unit) => globalSem.withPermits(1)(processEvaluation(unit)),
+      { concurrency: "unbounded", discard: true }
+    );
+  } else {
+    yield* Effect.forEach(
+      evaluationUnits,
+      processEvaluation,
+      maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+    );
+  }
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
     Ref.get(completedRef),
     Ref.get(passedRef),
@@ -935,144 +1163,53 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     artifactPath: task.snapshot.artifactPath
   };
   yield* updateSnapshot(task.runId, (snapshot) => ({
-    ...snapshot,
-    status: "completed",
-    completedTestCases: completedEvaluations,
-    passedTestCases: passedUniqueTestCases,
-    failedTestCases: failedUniqueTestCases,
-    finishedAt
-  }));
-  yield* publishEvent(completedEvent);
-  yield* Queue.offer(persistenceQueue, {
-    runId: task.runId,
-    artifactPath: task.snapshot.artifactPath,
-    payload: completedEvent
-  });
-  yield* publishEvent({
-    type: "ArtifactFlushed",
-    runId: task.runId,
-    artifactPath: task.snapshot.artifactPath
-  });
-});
-async function loadRunSnapshotsFromArtifacts(config) {
-  const baseDir = resolve(config.artifactDirectory);
-  let entries;
-  try {
-    entries = await readdir(baseDir);
-  } catch {
-    return [];
-  }
-  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
-  const snapshots = [];
-  for (const fileName of jsonlFiles) {
-    const filePath = join(baseDir, fileName);
-    try {
-      const snapshot = await parseArtifactToSnapshot(filePath, config);
-      if (snapshot) {
-        snapshots.push(snapshot);
-      }
-    } catch {
-    }
-  }
-  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
-}
-async function parseArtifactToSnapshot(filePath, _config) {
-  const content = await readFile(filePath, "utf8");
-  const lines = content.split("\n").filter((line) => line.trim().length > 0);
-  if (lines.length === 0) {
-    return null;
-  }
-  let runQueued = null;
-  let runCompleted = null;
-  let runFailed = null;
-  let runStarted = null;
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      const type = event.type;
-      if (type === "RunQueued") {
-        runQueued = {
-          runId: event.runId,
-          datasetId: event.datasetId,
-          datasetName: event.datasetName,
-          evaluatorIds: event.evaluatorIds,
-          totalTestCases: event.totalTestCases ?? 0,
-          artifactPath: event.artifactPath ?? filePath,
-          ts: event.ts
-        };
-      }
-      if (type === "RunStarted") {
-        runStarted = { startedAt: event.startedAt };
-      }
-      if (type === "RunCompleted") {
-        runCompleted = {
-          passedTestCases: event.passedTestCases,
-          failedTestCases: event.failedTestCases,
-          totalTestCases: event.totalTestCases,
-          finishedAt: event.finishedAt
-        };
-      }
-      if (type === "RunFailed") {
-        runFailed = {
-          finishedAt: event.finishedAt,
-          errorMessage: event.errorMessage
-        };
-      }
-    } catch {
-    }
+    ...snapshot,
+    status: "completed",
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
+    finishedAt
+  }));
+  yield* publishEvent(completedEvent);
+  yield* Queue.offer(persistenceQueue, {
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath,
+    payload: completedEvent
+  });
+  yield* publishEvent({
+    type: "ArtifactFlushed",
+    runId: task.runId,
+    artifactPath: task.snapshot.artifactPath
+  });
+});
+
+// src/runner/name-pattern.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
   }
-  if (!runQueued) {
-    return null;
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
   }
-  const artifactPath = filePath;
-  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
-  const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
-  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
-  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
-    runId: runQueued.runId,
-    datasetId: runQueued.datasetId,
-    datasetName: runQueued.datasetName,
-    evaluatorIds: runQueued.evaluatorIds,
-    queuedAt: runQueued.ts ?? 0,
-    startedAt: runStarted?.startedAt,
-    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
-    totalTestCases: runQueued.totalTestCases,
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
-    status,
-    artifactPath,
-    errorMessage: runFailed?.errorMessage
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
   };
 }
-function aggregateTestCaseProgress(lines) {
-  let completedTestCases = 0;
-  const testCasePassedBy = /* @__PURE__ */ new Map();
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      if (event.type === "TestCaseProgress") {
-        const ev = event;
-        completedTestCases = ev.completedTestCases ?? completedTestCases;
-        const id = ev.testCaseId;
-        const current = testCasePassedBy.get(id);
-        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
-      }
-    } catch {
-    }
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
   }
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const passed of testCasePassedBy.values()) {
-    if (passed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
   }
-  return { completedTestCases, passedTestCases, failedTestCases };
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
 async function appendJsonLine(artifactPath, payload) {
   await mkdir(dirname(artifactPath), { recursive: true });
@@ -1131,32 +1268,12 @@ function searchCollectedTestCases(all, query) {
 }
 
 // src/runner/api.ts
-function parseRegexLiteral(pattern) {
-  if (!pattern.startsWith("/")) {
-    return void 0;
-  }
-  const lastSlash = pattern.lastIndexOf("/");
-  if (lastSlash <= 0) {
-    return void 0;
-  }
-  return {
-    source: pattern.slice(1, lastSlash),
-    flags: pattern.slice(lastSlash + 1)
-  };
-}
-function createNameMatcher(pattern) {
-  const normalizedPattern = pattern.trim();
-  const regexLiteral = parseRegexLiteral(normalizedPattern);
-  if (regexLiteral) {
-    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
-    return (value) => regex.test(value);
+function normalizeRunRepetitions(value) {
+  const n = value ?? 1;
+  if (!Number.isInteger(n) || n < 1) {
+    throw new Error(`repetitions must be a positive integer, got ${String(value)}`);
   }
-  if (normalizedPattern.includes("*")) {
-    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-    const regex = new RegExp(`^${escaped}$`, "i");
-    return (value) => regex.test(value);
-  }
-  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+  return n;
 }
 function mergeRunnerOverrides(base, next) {
   if (!base) {
@@ -1191,6 +1308,7 @@ var EffectRunner = class {
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.runConfigsById = /* @__PURE__ */ new Map();
     this.schedulerFiber = Effect.runFork(this.createSchedulerEffect());
     this.persistenceFiber = Effect.runFork(
       createPersistenceWorker(this.persistenceQueue)
@@ -1231,6 +1349,137 @@ var EffectRunner = class {
       (item) => matcher(item.evaluator.getName() ?? "")
     );
   }
+  async collectRunConfigs() {
+    const runConfigs = await collectRunConfigsFromFiles(this.config.discovery);
+    this.runConfigsById.clear();
+    const byNameLower = /* @__PURE__ */ new Map();
+    for (const item of runConfigs) {
+      const id = item.runConfig.getName();
+      const lower = id.toLowerCase();
+      const prev = byNameLower.get(lower);
+      if (prev !== void 0 && prev.filePath !== item.filePath) {
+        throw new Error(
+          `Duplicate RunConfig name "${id}" (matches "${prev.runConfig.getName()}" case-insensitively): ${prev.filePath} and ${item.filePath}`
+        );
+      }
+      byNameLower.set(lower, item);
+      this.runConfigsById.set(id, item);
+    }
+    return runConfigs;
+  }
+  async resolveRunConfigByName(name) {
+    if (this.runConfigsById.size === 0) {
+      await this.collectRunConfigs();
+    }
+    const key = validateRunConfigName(name, `RunConfig "${name.trim()}"`);
+    const keyLower = key.toLowerCase();
+    const matches = Array.from(this.runConfigsById.values()).filter(
+      (item) => item.runConfig.getName().toLowerCase() === keyLower
+    );
+    if (matches.length === 0) {
+      return void 0;
+    }
+    if (matches.length > 1) {
+      throw new Error(
+        `Multiple RunConfigs named "${name}": ${matches.map((m) => m.filePath).join(", ")}`
+      );
+    }
+    return matches[0];
+  }
+  async expandRunConfigToJobs(collected) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const rcName = collected.runConfig.getName();
+    const jobs = [];
+    const runs = collected.runConfig.getRuns();
+    for (const [i, row] of runs.entries()) {
+      const dsCollected = Array.from(this.datasetsById.values()).find(
+        (d) => d.dataset === row.dataset
+      );
+      if (!dsCollected) {
+        throw new Error(
+          `RunConfig "${rcName}" run[${i}]: dataset "${row.dataset.getDisplayLabel()}" was not found among discovered dataset exports (import the same module instances the scanner loads).`
+        );
+      }
+      let evaluatorIds;
+      if ("evaluatorPattern" in row && typeof row.evaluatorPattern === "string") {
+        const matcher = createNameMatcher(row.evaluatorPattern);
+        const matched = Array.from(this.evaluatorsById.values()).filter(
+          (item) => matcher(item.evaluator.getName() ?? "")
+        );
+        if (matched.length === 0) {
+          throw new Error(
+            `RunConfig "${rcName}" run[${i}]: no evaluator matched pattern "${row.evaluatorPattern}"`
+          );
+        }
+        evaluatorIds = matched.map((item) => item.id);
+      } else {
+        const evaluators = row.evaluators;
+        evaluatorIds = [];
+        for (const ev of evaluators) {
+          const found = Array.from(this.evaluatorsById.values()).find(
+            (item) => item.evaluator === ev
+          );
+          if (!found) {
+            throw new Error(
+              `RunConfig "${rcName}" run[${i}]: evaluator "${getEvaluatorDisplayLabel(ev) ?? "unknown"}" was not found among discovered evaluator exports`
+            );
+          }
+          evaluatorIds.push(found.id);
+        }
+      }
+      const repetitions = "repetitions" in row && row.repetitions !== void 0 ? row.repetitions : 1;
+      jobs.push({
+        datasetId: dsCollected.id,
+        evaluatorIds,
+        runConfigName: rcName,
+        runConfigDisplayLabel: collected.runConfig.getDisplayLabel(),
+        runConfigTags: collected.runConfig.getTags(),
+        repetitions
+      });
+    }
+    return jobs;
+  }
+  async expandRunConfigNamesToJobs(names) {
+    const jobs = [];
+    for (const name of names) {
+      const collected = await this.resolveRunConfigByName(name);
+      if (!collected) {
+        const known = await this.collectRunConfigs();
+        const available = known.map((r) => r.runConfig.getName()).sort();
+        throw new Error(
+          available.length > 0 ? `RunConfig "${name}" not found. Available RunConfigs: ${available.join(", ")}` : `RunConfig "${name}" not found and no RunConfigs were discovered.`
+        );
+      }
+      jobs.push(...await this.expandRunConfigToJobs(collected));
+    }
+    return jobs;
+  }
+  async runDatasetJobsWithSharedConcurrency(request) {
+    const globalConcurrency = Math.max(1, request.globalConcurrency);
+    const sem = Effect.unsafeMakeSemaphore(globalConcurrency);
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const snapshots = [];
+    for (const job of request.jobs) {
+      snapshots.push(
+        await this.startDatasetRun({
+          datasetId: job.datasetId,
+          evaluatorIds: job.evaluatorIds,
+          triggerId,
+          maxConcurrency: this.config.maxConcurrency ?? 1,
+          globalEvaluationSemaphore: sem,
+          runConfigName: job.runConfigName,
+          runConfigTags: job.runConfigTags,
+          repetitions: job.repetitions
+        })
+      );
+    }
+    return snapshots;
+  }
   async searchTestCases(query) {
     const testCases = await collectTestCasesFromFiles(this.config.discovery);
     return searchCollectedTestCases(testCases, query);
@@ -1249,36 +1498,46 @@ var EffectRunner = class {
     );
   }
   async runDatasetWith(request) {
+    const runConfigName = validateRunConfigName(
+      request.runConfigName,
+      "runDatasetWith.runConfigName"
+    );
+    return this.startDatasetRun({
+      datasetId: request.datasetId,
+      evaluatorIds: request.evaluatorIds,
+      triggerId: request.triggerId,
+      maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
+      repetitions: request.repetitions,
+      runConfigName,
+      runConfigTags: request.runConfigTags
+    });
+  }
+  async startDatasetRun(params) {
     if (this.datasetsById.size === 0) {
       await this.collectDatasets();
     }
     if (this.evaluatorsById.size === 0) {
       await this.collectEvaluators();
     }
-    const dataset = this.datasetsById.get(request.datasetId);
+    const dataset = this.datasetsById.get(params.datasetId);
     if (!dataset) {
-      throw new Error(`Unknown dataset: ${request.datasetId}`);
+      throw new Error(`Unknown dataset: ${params.datasetId}`);
     }
-    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    const selectedEvaluators = params.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
     if (selectedEvaluators.length === 0) {
       throw new Error("No evaluators selected for run");
     }
-    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
-    const totalEvaluations = selectedTestCases.reduce(
-      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-      0
-    );
-    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const selectedTestCases = await this.collectDatasetTestCases(params.datasetId);
+    const repetitions = normalizeRunRepetitions(params.repetitions);
+    const totalEvaluations = selectedTestCases.length * repetitions;
+    const runConfigTags = [...params.runConfigTags ?? []];
+    const triggerId = params.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
-    const artifactPath = createArtifactPath(
-      this.config.artifactDirectory,
-      request.datasetId,
-      runId
-    );
+    const artifactPath = createArtifactPath(this.config.artifactDirectory, params.datasetId, runId);
     const snapshot = {
       runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetId: params.datasetId,
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
       totalTestCases: totalEvaluations,
@@ -1298,8 +1557,8 @@ var EffectRunner = class {
     const queuedEvent = {
       type: "RunQueued",
       runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetId: params.datasetId,
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       totalTestCases: totalEvaluations,
       artifactPath
@@ -1312,17 +1571,20 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
-    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
         triggerId,
-        datasetId: request.datasetId,
+        datasetId: params.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
         snapshot,
-        maxConcurrency
+        maxConcurrency: params.maxConcurrency,
+        globalEvaluationSemaphore: params.globalEvaluationSemaphore,
+        runConfigName: params.runConfigName,
+        runConfigTags,
+        repetitions
       })
     );
     return snapshot;
@@ -1401,6 +1663,8 @@ function getDefaultConcurrency() {
 function parseSimpleCliArgs(argv) {
   const args = {
     help: false,
+    ci: false,
+    runConfigNames: [],
     unknownArgs: []
   };
   let index = 0;
@@ -1414,18 +1678,26 @@ function parseSimpleCliArgs(argv) {
       args.help = true;
       continue;
     }
+    if (token === "--ci") {
+      args.ci = true;
+      continue;
+    }
     if ((token === "--dataset" || token === "--datasetName") && argv[index + 1]) {
       args.datasetName = argv[index + 1];
       index += 1;
       continue;
     }
-    if ((token === "--evaluator" || token === "--name") && argv[index + 1]) {
-      args.evaluatorPattern = argv[index + 1];
+    if ((token === "--run-config" || token === "--runConfig") && argv[index + 1]) {
+      const next = argv[index + 1];
+      if (typeof next === "string") {
+        args.runConfigNames.push(next);
+      }
       index += 1;
       continue;
     }
     if ((token === "--concurrency" || token === "-c") && argv[index + 1]) {
-      const n = parseInt(argv[index + 1], 10);
+      const nextConc = argv[index + 1];
+      const n = typeof nextConc === "string" ? parseInt(nextConc, 10) : Number.NaN;
       if (!Number.isNaN(n) && n >= 1) {
         args.concurrency = n;
       }
@@ -1439,16 +1711,12 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern> [--concurrency N]",
-    "  eval-agents-simple generate --dataset <datasetName>",
+    "  eval-agents-simple run --run-config <name> [--run-config <name> ...] [--concurrency N] [--ci]",
+    "  eval-agents-simple generate --dataset <datasetId>",
     "",
     "Options:",
-    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential.",
-    "",
-    "Pattern examples for --evaluator:",
-    "  score-evaluator       exact name (case-insensitive)",
-    '  "*score*"             wildcard pattern',
-    '  "/score/i"            regex literal'
+    "  --ci                  With run: exit with code 1 if any test case fails.",
+    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential."
   ].join("\n");
 }
 
@@ -1499,7 +1767,7 @@ function GenerateView({
       const payload = testCases.map((item) => {
         const tc = item.testCase;
         return {
-          name: item.testCase.getName(),
+          name: getTestCaseDisplayLabel(item.testCase),
           input: item.testCase.getInput(),
           output: typeof tc.getOutput === "function" ? tc.getOutput() : void 0
         };
@@ -1512,7 +1780,7 @@ function GenerateView({
       if (!cancelled) {
         setResult({
           count: payload.length,
-          datasetName: dataset.dataset.getName(),
+          datasetName: getDatasetDisplayLabel(dataset.dataset),
           outputPath
         });
         setTimeout(() => onComplete(), 200);
@@ -1565,7 +1833,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   }
   const testCases = await runner.collectDatasetTestCases(dataset.id);
   const payload = testCases.map((item) => ({
-    name: item.testCase.getName(),
+    name: getTestCaseDisplayLabel(item.testCase),
     input: item.testCase.getInput(),
     output: readOutput2(item.testCase)
   }));
@@ -1573,7 +1841,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   const outputPath = createOutputPath(absoluteDatasetPath);
   await writeFile(outputPath, `${JSON.stringify(payload, null, 2)}
 `, "utf8");
-  console.log(`Generated ${payload.length} test cases for dataset "${dataset.dataset.getName()}".`);
+  console.log(`Generated ${payload.length} test cases for dataset "${getDatasetDisplayLabel(dataset.dataset)}".`);
   console.log(`Wrote ${outputPath}`);
 }
 async function generateDatasetJsonCommandInk(runner, datasetName) {
@@ -1723,8 +1991,7 @@ function formatScorePart(item, _scoreToColor, options) {
 }
 function RunView({
   runner,
-  datasetName,
-  evaluatorPattern,
+  runConfigNames,
   concurrency,
   onComplete
 }) {
@@ -1737,30 +2004,30 @@ function RunView({
   const [summary, setSummary] = useState(null);
   const [evaluatorNameById, setEvaluatorNameById] = useState(/* @__PURE__ */ new Map());
   const runEval = useCallback(async () => {
-    const dataset = await runner.resolveDatasetByName(datasetName);
-    if (!dataset) {
-      const known = await runner.collectDatasets();
-      const available = known.map((item) => item.dataset.getName()).sort();
-      onComplete(
-        new Error(
-          available.length > 0 ? `Dataset "${datasetName}" not found. Available: ${available.join(", ")}` : `Dataset "${datasetName}" not found.`
-        )
-      );
+    const rcList = runConfigNames.filter((n) => n.trim().length > 0);
+    if (rcList.length === 0) {
+      onComplete(new Error("At least one RunConfig name is required."));
       return;
     }
-    const evaluators = await runner.resolveEvaluatorsByNamePattern(evaluatorPattern);
-    if (evaluators.length === 0) {
-      const known = await runner.collectEvaluators();
-      const available = known.map((item) => item.evaluator.getName()).filter((name) => typeof name === "string").sort();
-      onComplete(
-        new Error(
-          available.length > 0 ? `No evaluator matched "${evaluatorPattern}". Available: ${available.join(", ")}` : `No evaluator matched "${evaluatorPattern}".`
-        )
-      );
+    setStartedEvaluations(0);
+    setCompletedEvaluations(0);
+    setTestCases([]);
+    setRunningEvaluations([]);
+    setSummary(null);
+    let jobs;
+    try {
+      jobs = await runner.expandRunConfigNamesToJobs(rcList);
+    } catch (err) {
+      onComplete(err instanceof Error ? err : new Error(String(err)));
+      return;
+    }
+    if (jobs.length === 0) {
+      onComplete(new Error("No jobs expanded from RunConfigs."));
       return;
     }
+    const allEvaluators = await runner.collectEvaluators();
     const nameById = new Map(
-      evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
+      allEvaluators.map((item) => [item.id, getEvaluatorDisplayLabel(item.evaluator) ?? item.id])
     );
     setEvaluatorNameById(nameById);
     const aggregates = /* @__PURE__ */ new Map();
@@ -1768,21 +2035,30 @@ function RunView({
     let overallScoreTotal = 0;
     let overallScoreSumSq = 0;
     let overallScoreCount = 0;
-    const done = new Promise((resolve5) => {
+    const batchPendingRunIds = /* @__PURE__ */ new Set();
+    const runIdToLabel = /* @__PURE__ */ new Map();
+    let batchReady = false;
+    const completedRuns = /* @__PURE__ */ new Map();
+    const done = new Promise((resolve5, reject) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
+        if (batchReady && "runId" in event && typeof event.runId === "string" && !batchPendingRunIds.has(event.runId)) {
+          return;
+        }
         if (event.type === "TestCaseStarted") {
-          setStartedEvaluations(event.startedTestCases);
+          setStartedEvaluations((c) => c + 1);
           setRunningEvaluations((prev) => {
             const withoutDuplicate = prev.filter(
-              (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
+              (item) => !(item.testCaseId === event.testCaseId && item.repetitionIndex === event.repetitionIndex && item.runId === event.runId)
             );
             return [
               ...withoutDuplicate,
               {
+                runId: event.runId,
                 testCaseId: event.testCaseId,
                 name: event.testCaseName,
-                rerunIndex: event.rerunIndex,
-                rerunTotal: event.rerunTotal,
+                repetitionId: event.repetitionId,
+                repetitionIndex: event.repetitionIndex,
+                repetitionCount: event.repetitionCount,
                 startedTestCases: event.startedTestCases,
                 totalTestCases: event.totalTestCases
               }
@@ -1818,9 +2094,12 @@ function RunView({
               scoreItemsByEvaluatorScore.set(key, list);
             }
           }
+          const label = runIdToLabel.get(event.runId);
+          const compositeId = `${event.runId}:${event.testCaseId}`;
+          const displayName = label !== void 0 ? `${label} \u203A ${event.testCaseName}` : event.testCaseName;
           setTestCases((prev) => {
             const byId = new Map(prev.map((tc) => [tc.testCaseId, tc]));
-            const existing = byId.get(event.testCaseId);
+            const existing = byId.get(compositeId);
             const newEvent = {
               evaluatorScores: event.evaluatorScores.map((item) => ({
                 evaluatorId: item.evaluatorId,
@@ -1837,12 +2116,12 @@ function RunView({
             const isAggregated = events.length > 1;
             const aggregatedEvaluatorScores = aggregateEvaluatorScores(events, nameById);
             const merged = {
-              name: event.testCaseName,
-              testCaseId: event.testCaseId,
+              name: displayName,
+              testCaseId: compositeId,
               completedTestCases: event.completedTestCases,
               totalTestCases: event.totalTestCases,
-              rerunIndex: event.rerunIndex,
-              rerunTotal: event.rerunTotal,
+              repetitionIndex: event.repetitionIndex,
+              repetitionCount: event.repetitionCount,
               durationMs: events.reduce((s, e) => s + e.durationMs, 0),
               passed: events.every((e) => e.passed),
               errorMessage: event.errorMessage,
@@ -1850,84 +2129,118 @@ function RunView({
               aggregatedEvaluatorScores,
               isAggregated
             };
-            byId.set(event.testCaseId, merged);
-            setCompletedEvaluations(event.completedTestCases);
-            setRunningEvaluations(
-              (running) => running.filter(
-                (item) => !(item.testCaseId === event.testCaseId && item.rerunIndex === event.rerunIndex)
-              )
-            );
+            byId.set(compositeId, merged);
             return Array.from(byId.values());
           });
+          setCompletedEvaluations((c) => c + 1);
+          setRunningEvaluations(
+            (running) => running.filter(
+              (item) => !(item.testCaseId === event.testCaseId && item.repetitionIndex === event.repetitionIndex && item.runId === event.runId)
+            )
+          );
         }
-        if (event.type === "RunCompleted" || event.type === "RunFailed") {
+        if (event.type === "RunFailed") {
+          if (batchReady && !batchPendingRunIds.has(event.runId)) {
+            return;
+          }
           unsubscribe();
-          resolve5(event);
+          reject(new Error(`Run failed: ${event.errorMessage}`));
+          return;
+        }
+        if (event.type === "RunCompleted") {
+          if (!batchPendingRunIds.has(event.runId)) {
+            return;
+          }
+          completedRuns.set(event.runId, event);
+          batchPendingRunIds.delete(event.runId);
+          if (batchPendingRunIds.size === 0) {
+            unsubscribe();
+            resolve5();
+          }
         }
       });
     });
-    const snapshot = await runner.runDatasetWith({
-      datasetId: dataset.id,
-      evaluatorIds: evaluators.map((item) => item.id),
-      concurrency
+    const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
+      jobs,
+      globalConcurrency: concurrency
     });
+    for (let i = 0; i < snapshots.length; i += 1) {
+      const snap = snapshots[i];
+      const job = jobs[i];
+      if (snap && job) {
+        runIdToLabel.set(
+          snap.runId,
+          `${job.runConfigDisplayLabel ?? job.runConfigName} \xB7 ${snap.datasetName}`
+        );
+        batchPendingRunIds.add(snap.runId);
+      }
+    }
+    const totalUnits = snapshots.reduce((sum, s) => sum + s.totalTestCases, 0);
+    batchReady = true;
+    const runConfigLabels = await Promise.all(
+      rcList.map(async (n) => {
+        const collected = await runner.resolveRunConfigByName(n);
+        return collected?.runConfig.getDisplayLabel() ?? n;
+      })
+    );
     setRunInfo({
-      runId: snapshot.runId,
-      datasetName: snapshot.datasetName,
-      evaluatorNames: evaluators.map((e) => e.evaluator.getName() ?? e.id),
-      totalTestCases: snapshot.totalTestCases
+      names: runConfigLabels,
+      jobs: jobs.length,
+      totalTestCases: totalUnits
     });
     setPhase("running");
-    const finalEvent = await done;
-    if (finalEvent.type === "RunFailed") {
-      onComplete(new Error(`Run failed: ${finalEvent.errorMessage}`));
+    try {
+      await done;
+    } catch (err) {
+      onComplete(err instanceof Error ? err : new Error(String(err)));
       return;
     }
-    const completed = finalEvent;
+    let passedTestCases = 0;
+    let failedTestCases = 0;
+    let totalTestCases = 0;
+    const artifacts = [];
+    for (const ev of completedRuns.values()) {
+      passedTestCases += ev.passedTestCases;
+      failedTestCases += ev.failedTestCases;
+      totalTestCases += ev.totalTestCases;
+      artifacts.push(ev.artifactPath);
+    }
     setSummary({
-      passedTestCases: completed.passedTestCases,
-      failedTestCases: completed.failedTestCases,
-      totalTestCases: completed.totalTestCases,
+      passedTestCases,
+      failedTestCases,
+      totalTestCases,
       overallScoreTotal,
       overallScoreSumSq,
       overallScoreCount,
       aggregates: new Map(aggregates),
       scoreItemsByEvaluatorScore: new Map(scoreItemsByEvaluatorScore),
-      artifactPath: completed.artifactPath
+      artifactPath: artifacts.join("\n")
     });
     setPhase("completed");
-    setTimeout(() => onComplete(), 200);
-  }, [runner, datasetName, evaluatorPattern, concurrency, onComplete]);
+    const exitCode = failedTestCases > 0 ? 1 : 0;
+    setTimeout(() => onComplete(void 0, exitCode), 200);
+  }, [runner, runConfigNames, concurrency, onComplete]);
   useEffect(() => {
     void runEval();
   }, [runEval]);
   return /* @__PURE__ */ jsxs(Box, { flexDirection: "column", padding: 1, children: [
     /* @__PURE__ */ jsx(Box, { marginBottom: 1, children: /* @__PURE__ */ jsx(Banner, {}) }),
     runInfo && /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 1, children: [
-      /* @__PURE__ */ jsxs(Text, { children: [
-        /* @__PURE__ */ jsxs(Text, { color: "cyan", bold: true, children: [
-          "Run",
-          " "
-        ] }),
-        /* @__PURE__ */ jsx(Text, { color: "gray", children: runInfo.runId })
+      /* @__PURE__ */ jsxs(Text, { color: "cyan", bold: true, children: [
+        "RunConfigs",
+        " "
       ] }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: runInfo.names.join(", ") }),
       /* @__PURE__ */ jsxs(Text, { children: [
         /* @__PURE__ */ jsxs(Text, { color: "cyan", bold: true, children: [
-          "Dataset",
+          "Jobs",
           " "
         ] }),
-        runInfo.datasetName
+        runInfo.jobs
       ] }),
       /* @__PURE__ */ jsxs(Text, { children: [
         /* @__PURE__ */ jsxs(Text, { color: "cyan", bold: true, children: [
-          "Evaluators",
-          " "
-        ] }),
-        runInfo.evaluatorNames.join(", ")
-      ] }),
-      /* @__PURE__ */ jsxs(Text, { children: [
-        /* @__PURE__ */ jsxs(Text, { color: "cyan", bold: true, children: [
-          "Test cases",
+          "Evaluation units",
           " "
         ] }),
         runInfo.totalTestCases
@@ -1940,22 +2253,29 @@ function RunView({
           label: `Evaluations ${completedEvaluations}/${runInfo?.totalTestCases ?? 0} completed \u2022 ${startedEvaluations}/${runInfo?.totalTestCases ?? 0} started`
         }
       ),
-      runningEvaluations.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxs(Text, { color: "yellow", children: [
-        "[running ",
-        item.startedTestCases,
-        "/",
-        item.totalTestCases,
-        "] ",
-        item.name,
-        " ",
-        /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-          "(",
-          item.rerunIndex,
-          "/",
-          item.rerunTotal,
-          ")"
-        ] })
-      ] }, `${item.testCaseId}:${item.rerunIndex}`)) })
+      runningEvaluations.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginTop: 1, children: runningEvaluations.map((item) => /* @__PURE__ */ jsxs(
+        Text,
+        {
+          color: "yellow",
+          children: [
+            "[running ",
+            item.startedTestCases,
+            "/",
+            item.totalTestCases,
+            "] ",
+            item.name,
+            " ",
+            /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+              "(",
+              item.repetitionIndex,
+              "/",
+              item.repetitionCount,
+              ")"
+            ] })
+          ]
+        },
+        `${item.runId ?? ""}:${item.testCaseId}:${item.repetitionId}:${item.repetitionIndex}`
+      )) })
     ] }),
     testCases.length > 0 && /* @__PURE__ */ jsx(Box, { flexDirection: "column", marginBottom: 1, children: testCases.map((tc) => /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 0, children: [
       /* @__PURE__ */ jsxs(Text, { children: [
@@ -1971,9 +2291,9 @@ function RunView({
         " ",
         /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
           "(",
-          tc.rerunIndex,
+          tc.repetitionIndex,
           "/",
-          tc.rerunTotal,
+          tc.repetitionCount,
           ")"
         ] }),
         /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
@@ -2013,7 +2333,7 @@ function RunView({
             })
           ] }) : null
         ] }),
-        item.scores.length > 0 ? item.scores.map((s, idx) => {
+        item.scores.length > 0 ? item.scores.map((s) => {
           const def = s.def ?? getScoreById(s.id);
           const scoreLabel = s.name ?? def?.name ?? def?.id ?? s.id;
           return /* @__PURE__ */ jsxs(
@@ -2030,18 +2350,25 @@ function RunView({
                 })
               ]
             },
-            `${item.evaluatorId}-${s.id}-${idx}`
+            `${item.evaluatorId}-${s.id}-${scoreLabel}`
           );
         }) : /* @__PURE__ */ jsx(Text, { color: "gray", children: " n/a" }),
         !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsx(Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
-          (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getDiffLines(log).map(({ type, line }, lineIdx) => /* @__PURE__ */ jsx(
-            Text,
+          (log) => log.type === "diff" ? /* @__PURE__ */ jsx(
+            Box,
             {
-              color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
-              children: line
+              flexDirection: "column",
+              children: getDiffLines(log).map(({ type, line }) => /* @__PURE__ */ jsx(
+                Text,
+                {
+                  color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+                  children: line
+                },
+                `${type}:${line}`
+              ))
             },
-            lineIdx
-          )) }, logIdx) : log.type === "log" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getLogLines(log).map((line, lineIdx) => /* @__PURE__ */ jsx(Text, { color: "gray", children: line }, lineIdx)) }, logIdx) : null
+            `diff:${getDiffLines(log).map((x) => x.line).join("|")}`
+          ) : log.type === "log" ? /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: getLogLines(log).map((line) => /* @__PURE__ */ jsx(Text, { color: "gray", children: line }, line)) }, `log:${getLogLines(log).join("\n")}`) : null
         ) })
       ] }, item.evaluatorId))
     ] }, tc.testCaseId)) }),
@@ -2165,10 +2492,10 @@ function RunView({
           ] }, tc.testCaseId);
         })
       ] }),
-      /* @__PURE__ */ jsx(Box, { marginTop: 1, children: /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
-        "artifact: ",
-        summary.artifactPath
-      ] }) })
+      /* @__PURE__ */ jsxs(Box, { marginTop: 1, flexDirection: "column", children: [
+        /* @__PURE__ */ jsx(Text, { color: "gray", children: "artifact(s):" }),
+        summary.artifactPath.split("\n").map((line) => /* @__PURE__ */ jsx(Text, { color: "gray", children: line }, line))
+      ] })
     ] })
   ] });
 }
@@ -2380,25 +2707,14 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern, concurrency) {
-  const dataset = await runner.resolveDatasetByName(datasetName);
-  if (!dataset) {
-    const known = await runner.collectDatasets();
-    const available = known.map((item) => item.dataset.getName()).sort();
-    throw new Error(
-      available.length > 0 ? `Dataset "${datasetName}" not found. Available datasets: ${available.join(", ")}` : `Dataset "${datasetName}" not found and no datasets were discovered.`
-    );
-  }
-  const evaluators = await runner.resolveEvaluatorsByNamePattern(evaluatorPattern);
-  if (evaluators.length === 0) {
-    const known = await runner.collectEvaluators();
-    const available = known.map((item) => item.evaluator.getName()).filter((name) => typeof name === "string").sort();
-    throw new Error(
-      available.length > 0 ? `No evaluator matched "${evaluatorPattern}". Available evaluators: ${available.join(", ")}` : `No evaluator matched "${evaluatorPattern}" and no evaluators were discovered.`
-    );
+async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency) {
+  const jobs = await runner.expandRunConfigNamesToJobs(runConfigNames);
+  if (jobs.length === 0) {
+    throw new Error("No jobs expanded from RunConfigs.");
   }
+  const evaluators = await runner.collectEvaluators();
   const evaluatorNameById = new Map(
-    evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id])
+    evaluators.map((item) => [item.id, getEvaluatorDisplayLabel(item.evaluator) ?? item.id])
   );
   const aggregates = /* @__PURE__ */ new Map();
   const scoreItemsByEvaluatorScore = /* @__PURE__ */ new Map();
@@ -2406,11 +2722,11 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
   let overallScoreTotal = 0;
   let overallScoreSumSq = 0;
   let overallScoreCount = 0;
-  let startedCount = 0;
-  let completedCount = 0;
+  let globalStartedUnits = 0;
+  let globalCompletedUnits = 0;
   let totalCount = 0;
   let runFinished = false;
-  const inFlightReruns = /* @__PURE__ */ new Set();
+  const inFlightRepetitions = /* @__PURE__ */ new Set();
   const spinnerFrames = ["\u280B", "\u2819", "\u2838", "\u2834", "\u2826", "\u2807"];
   let spinnerIndex = 0;
   function clearLine() {
@@ -2432,33 +2748,46 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
     spinnerIndex += 1;
     process.stdout.write(
       `\r${colorize(frame, ansi2.cyan)} Running evaluations ${colorize(
-        `${completedCount}/${totalCount}`,
+        `${globalCompletedUnits}/${totalCount}`,
         ansi2.bold
-      )} completed ${colorize(`${startedCount}/${totalCount}`, ansi2.bold)} started ${colorize(`(${inFlightReruns.size} running)`, ansi2.dim)}`
+      )} completed ${colorize(`${globalStartedUnits}/${totalCount}`, ansi2.bold)} started ${colorize(`(${inFlightRepetitions.size} running)`, ansi2.dim)}`
     );
   }
   let lastPrintedTestCaseId = null;
   let lastPrintedLineCount = 0;
   let spinnerTimer;
-  const done = new Promise((resolve5) => {
+  const batchPendingRunIds = /* @__PURE__ */ new Set();
+  const runIdToLabel = /* @__PURE__ */ new Map();
+  let batchReady = false;
+  const completedRuns = /* @__PURE__ */ new Map();
+  const done = new Promise((resolve5, reject) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
+      if (batchReady && "runId" in event && typeof event.runId === "string" && !batchPendingRunIds.has(event.runId)) {
+        return;
+      }
+      const rowPrefix = typeof event.runId === "string" ? runIdToLabel.get(event.runId) : void 0;
+      const pfx = rowPrefix !== void 0 ? `${colorize(`[${rowPrefix}]`, ansi2.dim)} ` : "";
       if (event.type === "TestCaseStarted") {
-        startedCount = event.startedTestCases;
-        inFlightReruns.add(`${event.testCaseId}:${event.rerunIndex}`);
+        globalStartedUnits += 1;
+        inFlightRepetitions.add(
+          `${event.runId}:${event.testCaseId}:${event.repetitionId}:${event.repetitionIndex}`
+        );
         clearLine();
         process.stdout.write(
-          `${colorize(`[started ${event.startedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize("(running)", ansi2.dim)}
+          `${pfx}${colorize(`[started ${event.startedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.repetitionIndex}/${event.repetitionCount})`, ansi2.cyan)} ${colorize("(running)", ansi2.dim)}
 `
         );
         drawSpinner();
       }
       if (event.type === "TestCaseProgress") {
-        completedCount = event.completedTestCases;
-        inFlightReruns.delete(`${event.testCaseId}:${event.rerunIndex}`);
+        globalCompletedUnits += 1;
+        inFlightRepetitions.delete(
+          `${event.runId}:${event.testCaseId}:${event.repetitionId}:${event.repetitionIndex}`
+        );
         const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
         const averageScore = numericScores.length > 0 ? numericScores.reduce((sum, value) => sum + value, 0) / numericScores.length : void 0;
-        const testCaseId = event.testCaseId;
-        const existing = testCaseByTestId.get(testCaseId) ?? {
+        const compositeId = `${event.runId}:${event.testCaseId}`;
+        const existing = testCaseByTestId.get(compositeId) ?? {
           name: event.testCaseName,
           events: []
         };
@@ -2468,7 +2797,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
           durationMs: event.durationMs,
           evaluatorScores: event.evaluatorScores
         });
-        testCaseByTestId.set(testCaseId, existing);
+        testCaseByTestId.set(compositeId, existing);
         for (const item of event.evaluatorScores) {
           const numeric = toNumericScoreFromScores(item.scores);
           if (numeric !== void 0) {
@@ -2497,10 +2826,10 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
             scoreItemsByEvaluatorScore.set(key, list);
           }
         }
-        const isSameTestCase = lastPrintedTestCaseId === testCaseId;
-        const isLastRerun = event.rerunIndex >= event.rerunTotal;
+        const isSameTestCase = lastPrintedTestCaseId === compositeId;
+        const isLastRepetition = event.repetitionIndex >= event.repetitionCount;
         const isNonTty = !process.stdout.isTTY;
-        const skipPrintNonTty = isNonTty && event.rerunTotal > 1 && !isLastRerun;
+        const skipPrintNonTty = isNonTty && event.repetitionCount > 1 && !isLastRepetition;
         if (isSameTestCase && lastPrintedLineCount > 0 && !skipPrintNonTty) {
           cursorUp(lastPrintedLineCount);
         }
@@ -2511,7 +2840,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
         const lines = [];
         const statusSuffix = event.errorMessage ? ` ${colorize("ERROR", `${ansi2.bold}${ansi2.red}`)}` : "";
         lines.push(
-          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.rerunIndex}/${event.rerunTotal})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}${statusSuffix}`
+          `${pfx}${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi2.cyan)} ${event.testCaseName} ${colorize(`(${event.repetitionIndex}/${event.repetitionCount})`, ansi2.cyan)} ${colorize(`(${durationMs}ms)`, ansi2.dim)}${statusSuffix}`
         );
         if (event.errorMessage) {
           lines.push(colorize(event.errorMessage, ansi2.red));
@@ -2542,64 +2871,102 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
           }
         }
         if (!skipPrintNonTty) {
-          for (let i = 0; i < lines.length; i++) {
+          for (let i = 0; i < lines.length; i += 1) {
             process.stdout.write(`\r\x1B[2K${lines[i]}
 `);
           }
-          lastPrintedTestCaseId = testCaseId;
+          lastPrintedTestCaseId = compositeId;
           lastPrintedLineCount = lines.length;
         }
         drawSpinner();
       }
-      if (event.type === "RunCompleted" || event.type === "RunFailed") {
+      if (event.type === "RunFailed") {
+        if (batchReady && !batchPendingRunIds.has(event.runId)) {
+          return;
+        }
         runFinished = true;
         clearLine();
         unsubscribe();
-        resolve5(event);
+        reject(new Error(`Run failed: ${event.errorMessage}`));
+        return;
+      }
+      if (event.type === "RunCompleted") {
+        if (!batchPendingRunIds.has(event.runId)) {
+          return;
+        }
+        completedRuns.set(event.runId, event);
+        batchPendingRunIds.delete(event.runId);
+        if (batchPendingRunIds.size === 0) {
+          runFinished = true;
+          clearLine();
+          unsubscribe();
+          resolve5();
+        }
       }
     });
   });
-  const snapshot = await runner.runDatasetWith({
-    datasetId: dataset.id,
-    evaluatorIds: evaluators.map((item) => item.id),
-    concurrency
+  console.log(colorize("=== Eval Run Started (RunConfigs) ===", `${ansi2.bold}${ansi2.cyan}`));
+  for (const name of runConfigNames) {
+    const collected = await runner.resolveRunConfigByName(name);
+    const label = collected?.runConfig.getDisplayLabel() ?? name;
+    console.log(`RunConfig: ${colorize(label, ansi2.bold)}`);
+  }
+  console.log(`Jobs: ${colorize(String(jobs.length), ansi2.bold)}`);
+  console.log(`Shared concurrency: ${colorize(String(concurrency), ansi2.bold)}`);
+  console.log("");
+  const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
+    jobs,
+    globalConcurrency: concurrency
   });
-  totalCount = snapshot.totalTestCases;
-  console.log(colorize("=== Eval Run Started ===", `${ansi2.bold}${ansi2.cyan}`));
-  console.log(`Run: ${colorize(snapshot.runId, ansi2.cyan)}`);
-  console.log(`Dataset: ${colorize(snapshot.datasetName, ansi2.bold)}`);
-  console.log(
-    `Evaluators: ${evaluators.map((item) => item.evaluator.getName() ?? item.id).join(", ")}`
-  );
-  console.log(`Total test cases: ${colorize(String(snapshot.totalTestCases), ansi2.bold)}`);
+  for (let i = 0; i < snapshots.length; i += 1) {
+    const snap = snapshots[i];
+    const job = jobs[i];
+    if (snap && job) {
+      runIdToLabel.set(
+        snap.runId,
+        `${job.runConfigDisplayLabel ?? job.runConfigName} \xB7 ${snap.datasetName}`
+      );
+      batchPendingRunIds.add(snap.runId);
+    }
+  }
+  totalCount = snapshots.reduce((sum, s) => sum + s.totalTestCases, 0);
+  console.log(`Total evaluation units: ${colorize(String(totalCount), ansi2.bold)}`);
   console.log("");
+  batchReady = true;
   drawSpinner();
   spinnerTimer = setInterval(drawSpinner, 100);
-  const finalEvent = await done;
+  await done;
   if (spinnerTimer) {
     clearInterval(spinnerTimer);
   }
-  if (finalEvent.type === "RunFailed") {
-    throw new Error(`Run failed: ${finalEvent.errorMessage}`);
-  }
-  const completed = finalEvent;
   console.log("");
-  console.log(colorize("=== Run Summary ===", `${ansi2.bold}${ansi2.cyan}`));
-  console.log(
-    `- passed: ${colorize(`${completed.passedTestCases}/${completed.totalTestCases}`, ansi2.green)}`
-  );
-  console.log(
-    `- failed: ${colorize(
-      `${completed.failedTestCases}/${completed.totalTestCases}`,
-      completed.failedTestCases > 0 ? ansi2.red : ansi2.dim
-    )}`
-  );
+  console.log(colorize("=== Run Summary (all jobs) ===", `${ansi2.bold}${ansi2.cyan}`));
+  for (const snap of snapshots) {
+    const completed = completedRuns.get(snap.runId);
+    if (!completed) {
+      continue;
+    }
+    const label = runIdToLabel.get(snap.runId) ?? snap.runId;
+    console.log("");
+    console.log(colorize(`\u2014 ${label}`, ansi2.magenta));
+    console.log(
+      `- passed: ${colorize(`${completed.passedTestCases}/${completed.totalTestCases}`, ansi2.green)}`
+    );
+    console.log(
+      `- failed: ${colorize(
+        `${completed.failedTestCases}/${completed.totalTestCases}`,
+        completed.failedTestCases > 0 ? ansi2.red : ansi2.dim
+      )}`
+    );
+    console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
+  }
   if (overallScoreCount > 0) {
     const overallAverage = overallScoreTotal / overallScoreCount;
     const overallSd = sampleStdDev2(overallScoreTotal, overallScoreSumSq, overallScoreCount);
     const avgStr = overallSd !== void 0 ? `${overallAverage.toFixed(2)} \xB1 ${overallSd.toFixed(2)}` : overallAverage.toFixed(2);
+    console.log("");
     console.log(
-      `- overall avg score: ${colorize(
+      `- overall avg score (all jobs): ${colorize(
         avgStr,
         scoreToColor(overallAverage)
       )} ${colorize(createBar2(overallAverage), ansi2.dim)}`
@@ -2640,22 +3007,28 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern,
       );
     }
   }
-  console.log(`- artifact: ${colorize(completed.artifactPath, ansi2.dim)}`);
+  let failedTestCasesTotal = 0;
+  for (const snap of snapshots) {
+    const completed = completedRuns.get(snap.runId);
+    if (completed) {
+      failedTestCasesTotal += completed.failedTestCases;
+    }
+  }
+  return failedTestCasesTotal > 0 ? 1 : 0;
 }
-async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern, concurrency) {
+async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency) {
   return new Promise((resolve5, reject) => {
     const app = render(
       React.createElement(RunView, {
         runner,
-        datasetName,
-        evaluatorPattern,
+        runConfigNames,
         concurrency,
-        onComplete: (err) => {
+        onComplete: (err, exitCode) => {
           app.unmount();
           if (err) {
             reject(err);
           } else {
-            resolve5();
+            resolve5(exitCode ?? 0);
           }
         }
       })
@@ -2681,12 +3054,22 @@ async function main() {
   if (!args.command) {
     printUsageAndExit(1);
   }
-  if (!args.datasetName) {
-    console.error("Missing required --dataset <datasetName> argument.");
-    printUsageAndExit(1);
+  if (args.command === "run") {
+    if (args.runConfigNames.length === 0) {
+      console.error(
+        "Missing required --run-config <name> (repeat the flag to queue multiple RunConfigs)."
+      );
+      printUsageAndExit(1);
+    }
+    if (args.datasetName !== void 0) {
+      console.error(
+        "The run command no longer accepts --dataset; use --run-config <RunConfig name>."
+      );
+      printUsageAndExit(1);
+    }
   }
-  if (args.command === "run" && !args.evaluatorPattern) {
-    console.error("Missing required --evaluator <name-or-pattern> argument.");
+  if (args.command === "generate" && args.runConfigNames.length > 0) {
+    console.error("generate does not accept --run-config.");
     printUsageAndExit(1);
   }
   const useInk = process.stdout.isTTY === true;
@@ -2697,17 +3080,24 @@ async function main() {
   try {
     if (args.command === "run") {
       const concurrency = args.concurrency ?? getDefaultConcurrency();
-      await (useInk ? runSimpleEvalCommandInk : runSimpleEvalCommandPlain)(
+      const exitCode = await (useInk ? runSimpleEvalRunConfigsInk : runSimpleEvalRunConfigsPlain)(
         runner,
-        args.datasetName,
-        args.evaluatorPattern,
+        args.runConfigNames,
         concurrency
       );
+      if (args.ci && exitCode !== 0) {
+        process.exit(1);
+      }
       return;
     }
+    const genDataset = args.datasetName;
+    if (!genDataset) {
+      console.error("Missing required --dataset <datasetId> argument.");
+      printUsageAndExit(1);
+    }
     await (useInk ? generateDatasetJsonCommandInk : generateDatasetJsonCommandPlain)(
       runner,
-      args.datasetName
+      genDataset
     );
   } finally {
     await runner.shutdown();