@m4trix/evals 0.25.0 → 0.26.0

package/dist/cli.js

@@ -1,18 +1,18 @@
 #!/usr/bin/env node
 import { withFullScreen, useScreenSize } from 'fullscreen-ink';
-import React2, { useState, useRef, useReducer, useEffect, useMemo } from 'react';
+import React, { useState, useRef, useReducer, useEffect, useMemo } from 'react';
 import { useApp, useInput, Box, Text } from 'ink';
 import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
-import { resolve, relative, join, dirname } from 'path';
-import { LineGraph } from '@pppp606/ink-chart';
+import { Schema, Effect, PubSub, Queue, Ref, Fiber, Either, ParseResult } from 'effect';
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Ref, Fiber } from 'effect';
+import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
+import { resolve, join, relative, dirname } from 'path';
 import { existsSync } from 'fs';
 import * as jitiModule from 'jiti';
-import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
 import { diffLines } from 'diff';
 import stringify from 'fast-json-stable-stringify';
+import { LineGraph } from '@pppp606/ink-chart';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -90,11 +90,7 @@ function getFooterText(state) {
   }
   return "\u2191\u2193 move  Enter add/remove  S start run  / search  Esc cancel  q quit";
 }
-function ListItem({
-  selected,
-  label,
-  itemKey
-}) {
+function ListItem({ selected, label, itemKey }) {
   return /* @__PURE__ */ jsxs(Text, { color: selected ? "cyan" : "gray", bold: selected, children: [
     selected ? "\u25B8 " : "  ",
     label
@@ -121,9 +117,7 @@ function Pane({
     }
   );
 }
-function SectionHeader({
-  children
-}) {
+function SectionHeader({ children }) {
   return /* @__PURE__ */ jsx(Text, { color: "cyan", bold: true, children });
 }
 function StatusText({ status }) {
@@ -135,10 +129,7 @@ function StatusText({ status }) {
   ] });
 }
 var LEFT_PANE_WIDTH = 44;
-function RunsSidebar({
-  state,
-  runs
-}) {
+function RunsSidebar({ state, runs }) {
   const focused = state.focus === "left";
   return /* @__PURE__ */ jsxs(Pane, { width: LEFT_PANE_WIDTH, focused, children: [
     /* @__PURE__ */ jsx(SectionHeader, { children: "Runs" }),
@@ -167,11 +158,7 @@ function RunsSidebar({
   ] });
 }
 var BLOCKS = ["\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"];
-function Sparkline({
-  data,
-  width,
-  label
-}) {
+function Sparkline({ data, width, label }) {
   if (data.length === 0)
     return null;
   const max = Math.max(...data);
@@ -250,6 +237,50 @@ function isPrintableCharacter(input) {
 function isBackKey(key) {
   return key.backspace || key.delete;
 }
+var ENTITY_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
+function makeEntityIdSchema(brand, label) {
+  return Schema.String.pipe(
+    Schema.trimmed(),
+    Schema.minLength(1, {
+      message: () => `${label} must be non-empty.`
+    }),
+    Schema.pattern(ENTITY_ID_PATTERN, {
+      message: () => `${label} may only contain letters, digits, underscores, and hyphens (no spaces). Examples: "my-nightly", "my_nightly", "myNightly".`
+    }),
+    Schema.brand(brand)
+  );
+}
+var RunConfigNameSchema = makeEntityIdSchema("RunConfigName", "RunConfig name");
+makeEntityIdSchema("EvaluatorName", "Evaluator name");
+makeEntityIdSchema("TestCaseName", "Test case name");
+function validateWithSchema(schema, raw, context) {
+  const trimmed = raw.trim();
+  const decode = Schema.decodeUnknownEither(
+    schema
+  );
+  const result = decode(trimmed);
+  if (Either.isLeft(result)) {
+    throw new Error(`${context}: ${ParseResult.TreeFormatter.formatErrorSync(result.left)}`);
+  }
+  return result.right;
+}
+function validateRunConfigName(raw, context) {
+  return validateWithSchema(RunConfigNameSchema, raw, context);
+}
+
+// src/evals/evaluator.ts
+function getEvaluatorDisplayLabel(evaluator) {
+  if (typeof evaluator.getDisplayLabel === "function") {
+    const label = evaluator.getDisplayLabel();
+    if (label !== void 0) {
+      return label;
+    }
+  }
+  return typeof evaluator.getName === "function" ? evaluator.getName() : void 0;
+}
+function getEvaluatorTagList(evaluator) {
+  return typeof evaluator.getTags === "function" ? [...evaluator.getTags()] : [];
+}
 
 // src/cli/data.mock.json
 var data_mock_default = {
@@ -401,9 +432,7 @@ var data_mock_default = {
             { name: "contract_match", score: 100 },
             { name: "arg_validity", score: 100 }
           ],
-          checks: [
-            { name: "tool_calls", passed: true, detail: "0 unexpected" }
-          ],
+          checks: [{ name: "tool_calls", passed: true, detail: "0 unexpected" }],
           failures: [],
           meta: {
             model: "gpt-4o-mini",
@@ -426,9 +455,21 @@ var data_mock_default = {
     }
   ],
   evaluators: [
-    { id: "json-schema-validator", name: "JSON Schema Validator", configPreview: "strict=true" },
-    { id: "tool-call-contract-checker", name: "Tool-call Contract Checker", configPreview: "unexpectedCalls=error" },
-    { id: "rubric-judge", name: "Rubric Judge (LLM)", configPreview: "model=gpt-4o-mini; scale=0-100" },
+    {
+      id: "json-schema-validator",
+      name: "JSON Schema Validator",
+      configPreview: "strict=true"
+    },
+    {
+      id: "tool-call-contract-checker",
+      name: "Tool-call Contract Checker",
+      configPreview: "unexpectedCalls=error"
+    },
+    {
+      id: "rubric-judge",
+      name: "Rubric Judge (LLM)",
+      configPreview: "model=gpt-4o-mini; scale=0-100"
+    },
     { id: "pii-leak-detector", name: "PII Leak Detector", configPreview: "redact=false" }
   ]
 };
@@ -495,7 +536,7 @@ function toEvalDataset(item, snapshots) {
 function toEvaluatorOption(item) {
   return {
     id: item.id,
-    name: item.evaluator.getName() ?? toSlug(item.id),
+    name: getEvaluatorDisplayLabel(item.evaluator) ?? toSlug(item.id),
     configPreview: `Source: ${item.filePath}`
   };
 }
@@ -508,9 +549,7 @@ async function loadRunnerData(runner) {
   const memSnapshots = runner.getAllRunSnapshots();
   const seen = new Set(memSnapshots.map((s) => s.runId));
   const fromDisk = diskSnapshots.filter((s) => !seen.has(s.runId));
-  const snapshots = [...memSnapshots, ...fromDisk].sort(
-    (a, b) => b.queuedAt - a.queuedAt
-  );
+  const snapshots = [...memSnapshots, ...fromDisk].sort((a, b) => b.queuedAt - a.queuedAt);
   if (datasets.length === 0 && evaluators.length === 0) {
     return loadMockData();
   }
@@ -632,7 +671,11 @@ function reduceCliState(state, action) {
       return { ...state, overviewScrollOffset: Math.max(0, state.overviewScrollOffset - 1) };
     }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1), overviewScrollOffset: 0 };
+      return {
+        ...state,
+        datasetMenuIndex: Math.max(0, state.datasetMenuIndex - 1),
+        overviewScrollOffset: 0
+      };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.max(0, state.runMenuIndex - 1) };
@@ -650,10 +693,17 @@ function reduceCliState(state, action) {
       return { ...state, detailsScrollOffset: Math.min(action.max, state.detailsScrollOffset + 1) };
     }
     if (state.level === "datasets" && state.focus === "right") {
-      return { ...state, overviewScrollOffset: Math.min(action.max, state.overviewScrollOffset + 1) };
+      return {
+        ...state,
+        overviewScrollOffset: Math.min(action.max, state.overviewScrollOffset + 1)
+      };
     }
     if (state.level === "datasets") {
-      return { ...state, datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1), overviewScrollOffset: 0 };
+      return {
+        ...state,
+        datasetMenuIndex: Math.min(action.max, state.datasetMenuIndex + 1),
+        overviewScrollOffset: 0
+      };
     }
     if (state.level === "runs") {
       return { ...state, runMenuIndex: Math.min(action.max, state.runMenuIndex + 1) };
@@ -729,24 +779,168 @@ function reduceCliState(state, action) {
   }
   return state;
 }
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
+      }
+    } catch {
+    }
+  }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
+async function parseArtifactFile(artifactPath) {
+  try {
+    const content = await readFile(artifactPath, "utf8");
+    const lines = content.split("\n").filter((line) => line.trim().length > 0);
+    const results = [];
+    for (const line of lines) {
+      try {
+        const event = JSON.parse(line);
+        if (event.type === "TestCaseProgress") {
+          const ev = event;
+          const repetitionIndex = ev.repetitionIndex ?? ev.rerunIndex;
+          const repetitionCount = ev.repetitionCount ?? ev.rerunTotal;
+          results.push({
+            testCaseId: ev.testCaseId,
+            testCaseName: ev.testCaseName,
+            completedTestCases: ev.completedTestCases,
+            totalTestCases: ev.totalTestCases,
+            repetitionId: ev.repetitionId,
+            repetitionIndex,
+            repetitionCount,
+            passed: ev.passed,
+            durationMs: ev.durationMs,
+            evaluatorScores: ev.evaluatorScores ?? []
+          });
+        }
+      } catch {
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
 
 // src/runner/config.ts
 var defaultRunnerConfig = {
   discovery: {
     rootDir: process.cwd(),
     datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
-    evaluatorSuffixes: [
-      ".evaluator.ts",
-      ".evaluator.tsx",
-      ".evaluator.js",
-      ".evaluator.mjs"
-    ],
-    testCaseSuffixes: [
-      ".test-case.ts",
-      ".test-case.tsx",
-      ".test-case.js",
-      ".test-case.mjs"
-    ],
+    evaluatorSuffixes: [".evaluator.ts", ".evaluator.tsx", ".evaluator.js", ".evaluator.mjs"],
+    runConfigSuffixes: [".run-config.ts", ".run-config.tsx", ".run-config.js", ".run-config.mjs"],
+    testCaseSuffixes: [".test-case.ts", ".test-case.tsx", ".test-case.js", ".test-case.mjs"],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
   artifactDirectory: ".eval-results",
@@ -771,6 +965,11 @@ function toRunnerConfigOverrides(config) {
   } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
     discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
   }
+  if (rawDiscovery?.runConfigFilePatterns !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigFilePatterns;
+  } else if (rawDiscovery?.runConfigSuffixes !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigSuffixes;
+  }
   if (rawDiscovery?.testCaseFilePatterns !== void 0) {
     discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
   } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
@@ -813,14 +1012,15 @@ function getJitiLoader() {
   }
   const createJiti2 = jitiModule.createJiti ?? jitiModule.default;
   if (typeof createJiti2 !== "function") {
-    throw new Error(
-      "Failed to initialize jiti for m4trix eval config loading."
-    );
+    throw new Error("Failed to initialize jiti for m4trix eval config loading.");
   }
-  cachedLoader = createJiti2(import.meta.url, {
-    interopDefault: true,
-    moduleCache: true
-  });
+  cachedLoader = createJiti2(
+    import.meta.url,
+    {
+      interopDefault: true,
+      moduleCache: true
+    }
+  );
   return cachedLoader;
 }
 function resolveConfigModuleExport(loadedModule) {
@@ -868,6 +1068,9 @@ function isDatasetLike(value) {
 function isEvaluatorLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
 }
+function isRunConfigLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getRuns") && typeof value.getRuns === "function";
+}
 function isTestCaseLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
 }
@@ -924,9 +1127,7 @@ async function loadModuleExports(filePath) {
 }
 async function collectDatasetsFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.datasetSuffixes)
-  );
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.datasetSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -943,9 +1144,7 @@ async function collectDatasetsFromFiles(config) {
 }
 async function collectEvaluatorsFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes)
-  );
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.evaluatorSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -960,11 +1159,26 @@ async function collectEvaluatorsFromFiles(config) {
   );
   return found.flat();
 }
-async function collectTestCasesFromFiles(config) {
+async function collectRunConfigsFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
-  const matched = files.filter(
-    (filePath) => hasOneSuffix(filePath, config.testCaseSuffixes)
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.runConfigSuffixes));
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const runConfigs = exports.filter(isRunConfigLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return runConfigs.map((runConfig) => ({
+        id: runConfig.getName(),
+        filePath: relPath,
+        runConfig
+      }));
+    })
   );
+  return found.flat();
+}
+async function collectTestCasesFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.testCaseSuffixes));
   const found = await Promise.all(
     matched.map(async (absolutePath) => {
       const exports = await loadModuleExports(absolutePath);
@@ -1036,16 +1250,8 @@ function createDiffString(expected, actual, diffOptions) {
   const expectedProcessed = preprocessForDiff(expected, diffOptions);
   const actualProcessed = preprocessForDiff(actual, diffOptions);
   if (diffOptions?.keysOnly) {
-    const expectedKeys = JSON.stringify(
-      extractKeys(expectedProcessed),
-      null,
-      2
-    );
-    const actualKeys = JSON.stringify(
-      extractKeys(actualProcessed),
-      null,
-      2
-    );
+    const expectedKeys = JSON.stringify(extractKeys(expectedProcessed), null, 2);
+    const actualKeys = JSON.stringify(extractKeys(actualProcessed), null, 2);
     const parts2 = diffLines(expectedKeys, actualKeys);
     return formatDiffParts(parts2);
   }
@@ -1056,9 +1262,7 @@ function createDiffString(expected, actual, diffOptions) {
   }
   const parts = diffLines(expectedStr, actualStr);
   if (diffOptions?.outputNewOnly) {
-    const filtered = parts.filter(
-      (p) => p.added === true
-    );
+    const filtered = parts.filter((p) => p.added === true);
     return formatDiffParts(filtered);
   }
   return formatDiffParts(parts);
@@ -1125,6 +1329,17 @@ function getDiffLines(entry) {
   });
 }
 
+// src/evals/test-case.ts
+function getTestCaseDisplayLabel(testCase) {
+  if (typeof testCase.getDisplayLabel === "function") {
+    return testCase.getDisplayLabel();
+  }
+  return typeof testCase.getName === "function" ? testCase.getName() : "";
+}
+function getTestCaseTagList(testCase) {
+  return typeof testCase.getTags === "function" ? [...testCase.getTags()] : [];
+}
+
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
 var Metric = {
@@ -1148,29 +1363,74 @@ function getMetricById(id) {
   return registry.get(id);
 }
 
-// src/evals/score.ts
-var registry2 = /* @__PURE__ */ new Map();
-function formatScoreData(def, data, options) {
-  return options?.isAggregated ? def.formatAggregate(data) : def.formatValue(data);
+// src/evals/aggregators.ts
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
 }
-var ScoreAggregate = {
-  /** Average numeric fields. Use for scores like { value, delta }. */
-  averageFields(fields) {
-    return (values) => {
-      const count = values.length || 1;
-      const result = {};
-      for (const field of fields) {
-        result[field] = values.reduce(
-          (s, v) => s + (v[field] ?? 0),
-          0
-        ) / count;
-      }
-      return result;
-    };
-  },
-  /** Average selected numeric fields, with sample std dev tracked for `value`. */
-  averageWithVariance(fields) {
-    return (values) => {
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
+// src/evals/score.ts
+var registry2 = /* @__PURE__ */ new Map();
+function formatScoreData(def, data, options) {
+  return options?.isAggregated ? def.formatAggregate(data) : def.formatValue(data);
+}
+var ScoreAggregate = {
+  /** Average numeric fields. Use for scores like { value, delta }. */
+  averageFields(fields) {
+    return (values) => {
+      const count = values.length || 1;
+      const result = {};
+      for (const field of fields) {
+        result[field] = values.reduce((s, v) => s + (v[field] ?? 0), 0) / count;
+      }
+      return result;
+    };
+  },
+  /** Average selected numeric fields, with sample std dev tracked for `value`. */
+  averageWithVariance(fields) {
+    return (values) => {
       const count = values.length;
       const result = {};
       for (const field of fields) {
@@ -1197,13 +1457,10 @@ var ScoreAggregate = {
           (s, v) => s + (v[valueField] ?? 0),
           0
         );
-        const sumSq = values.reduce(
-          (s, v) => {
-            const value = v[valueField] ?? 0;
-            return s + value * value;
-          },
-          0
-        );
+        const sumSq = values.reduce((s, v) => {
+          const value = v[valueField] ?? 0;
+          return s + value * value;
+        }, 0);
         const mean = sum / count;
         const variance = (sumSq - count * mean * mean) / (count - 1);
         stdDev = variance > 0 ? Math.sqrt(variance) : 0;
@@ -1262,54 +1519,6 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
-// src/evals/aggregators.ts
-function aggregateTokenCountSum(values) {
-  const initial = {
-    input: 0,
-    output: 0,
-    inputCached: 0,
-    outputCached: 0
-  };
-  return values.reduce(
-    (acc, v) => ({
-      input: acc.input + (v.input ?? 0),
-      output: acc.output + (v.output ?? 0),
-      inputCached: acc.inputCached + (v.inputCached ?? 0),
-      outputCached: acc.outputCached + (v.outputCached ?? 0)
-    }),
-    initial
-  );
-}
-function aggregateLatencyAverage(values) {
-  if (values.length === 0) {
-    return { ms: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.ms, 0);
-  return { ms: sum / values.length };
-}
-
-// src/evals/metrics/standard.ts
-Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  aggregate: aggregateTokenCountSum,
-  format: (data, options) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    const base = `in:${input} out:${output} cached:${cached}`;
-    return options?.isAggregated ? `Total: ${base}` : base;
-  }
-});
-Metric.of({
-  id: "latency",
-  name: "Latency",
-  aggregate: aggregateLatencyAverage,
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
-});
-
 // src/evals/scores/standard.ts
 Score.of({
   id: "percent",
@@ -1416,15 +1625,17 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
-function buildEvaluationUnits(testCases) {
+function buildEvaluationUnits(testCases, repetitionCount) {
+  const count = Math.max(1, repetitionCount);
   const units = [];
   for (const testCaseItem of testCases) {
-    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    for (let r = 0; r < rerunTotal; r++) {
+    const repetitionId = `rep-${randomUUID()}`;
+    for (let r = 0; r < count; r++) {
       units.push({
         testCaseItem,
-        rerunIndex: r + 1,
-        rerunTotal
+        repetitionId,
+        repetitionIndex: r + 1,
+        repetitionCount: count
       });
     }
   }
@@ -1434,29 +1645,24 @@ function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
 function createArtifactPath(artifactDirectory, datasetId, runId) {
-  return join(
-    artifactDirectory,
-    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
-  );
+  return join(artifactDirectory, `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`);
 }
 function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
-  const { testCaseItem, rerunIndex, rerunTotal } = unit;
+  const { testCaseItem, repetitionId, repetitionIndex, repetitionCount } = unit;
   return Effect.gen(function* () {
     const evaluatorRunId = `run-${randomUUID()}`;
     const started = Date.now();
-    const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
-      n + 1,
-      n + 1
-    ]);
+    const startedEvaluations = yield* Ref.modify(startedRef, (n) => [n + 1, n + 1]);
     yield* publishEvent({
       type: "TestCaseStarted",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       startedTestCases: startedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal
+      repetitionId,
+      repetitionIndex,
+      repetitionCount
     });
     const evaluatorScores = [];
     let testCaseError;
@@ -1480,9 +1686,7 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
         return error;
       };
       try {
-        const ctx = yield* Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
+        const ctx = yield* Effect.promise(() => Promise.resolve(evaluator.resolveContext()));
         const result = yield* Effect.promise(
           () => Promise.resolve().then(
             () => evaluateFn({
@@ -1492,8 +1696,15 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
               meta: {
                 triggerId: task.triggerId,
                 runId: evaluatorRunId,
-                datasetId: task.datasetId
+                datasetId: task.datasetId,
+                repetitionId,
+                repetitionIndex,
+                repetitionCount,
+                runConfigName: task.runConfigName
               },
+              testCaseTags: getTestCaseTagList(testCaseItem.testCase),
+              runConfigTags: task.runConfigTags,
+              evaluatorTags: getEvaluatorTagList(evaluator),
               logDiff,
               log,
               createError
@@ -1536,21 +1747,19 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
         });
       }
     }
-    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
-    const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
-      n + 1,
-      n + 1
-    ]);
+    const repetitionPassedThis = evaluatorScores.every((s) => s.passed);
+    const completedEvaluations = yield* Ref.modify(completedRef, (n) => [n + 1, n + 1]);
     const progressEvent = {
       type: "TestCaseProgress",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       completedTestCases: completedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal,
-      passed: rerunPassedThis,
+      repetitionId,
+      repetitionIndex,
+      repetitionCount,
+      passed: repetitionPassedThis,
       durationMs: Date.now() - started,
       evaluatorScores,
       output,
@@ -1571,9 +1780,9 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       (map) => {
         const key = testCaseItem.id;
         const existing = map.get(key) ?? { completedCount: 0, results: [] };
-        const newResults = [...existing.results, rerunPassedThis];
+        const newResults = [...existing.results, repetitionPassedThis];
         const newCompletedCount = existing.completedCount + 1;
-        const isLast = newCompletedCount === rerunTotal;
+        const isLast = newCompletedCount === repetitionCount;
         const newMap = new Map(map);
         newMap.set(key, {
           completedCount: newCompletedCount,
@@ -1589,10 +1798,7 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       } else {
         yield* Ref.update(failedRef, (n) => n + 1);
       }
-      const [passed, failed] = yield* Effect.all([
-        Ref.get(passedRef),
-        Ref.get(failedRef)
-      ]);
+      const [passed, failed] = yield* Effect.all([Ref.get(passedRef), Ref.get(failedRef)]);
       yield* updateSnapshot(task.runId, (snapshot) => ({
         ...snapshot,
         passedTestCases: passed,
@@ -1613,10 +1819,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  const totalEvaluations = task.testCases.reduce(
-    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-    0
-  );
+  const totalEvaluations = task.testCases.length * Math.max(1, task.repetitions);
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* Ref.make(0);
   const startedRef = yield* Ref.make(0);
@@ -1625,7 +1828,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   const testCaseResultsRef = yield* Ref.make(
     /* @__PURE__ */ new Map()
   );
-  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const evaluationUnits = buildEvaluationUnits(task.testCases, task.repetitions);
   const processEvaluation = (unit) => processOneEvaluation(
     task,
     unit,
@@ -1639,11 +1842,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     failedRef,
     testCaseResultsRef
   );
-  yield* Effect.forEach(
-    evaluationUnits,
-    processEvaluation,
-    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
-  );
+  const globalSem = task.globalEvaluationSemaphore;
+  if (globalSem !== void 0) {
+    yield* Effect.forEach(
+      evaluationUnits,
+      (unit) => globalSem.withPermits(1)(processEvaluation(unit)),
+      { concurrency: "unbounded", discard: true }
+    );
+  } else {
+    yield* Effect.forEach(
+      evaluationUnits,
+      processEvaluation,
+      maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+    );
+  }
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
     Ref.get(completedRef),
     Ref.get(passedRef),
@@ -1679,155 +1891,34 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     artifactPath: task.snapshot.artifactPath
   });
 });
-async function loadRunSnapshotsFromArtifacts(config) {
-  const baseDir = resolve(config.artifactDirectory);
-  let entries;
-  try {
-    entries = await readdir(baseDir);
-  } catch {
-    return [];
-  }
-  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
-  const snapshots = [];
-  for (const fileName of jsonlFiles) {
-    const filePath = join(baseDir, fileName);
-    try {
-      const snapshot = await parseArtifactToSnapshot(filePath, config);
-      if (snapshot) {
-        snapshots.push(snapshot);
-      }
-    } catch {
-    }
-  }
-  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
-}
-async function parseArtifactToSnapshot(filePath, _config) {
-  const content = await readFile(filePath, "utf8");
-  const lines = content.split("\n").filter((line) => line.trim().length > 0);
-  if (lines.length === 0) {
-    return null;
-  }
-  let runQueued = null;
-  let runCompleted = null;
-  let runFailed = null;
-  let runStarted = null;
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      const type = event.type;
-      if (type === "RunQueued") {
-        runQueued = {
-          runId: event.runId,
-          datasetId: event.datasetId,
-          datasetName: event.datasetName,
-          evaluatorIds: event.evaluatorIds,
-          totalTestCases: event.totalTestCases ?? 0,
-          artifactPath: event.artifactPath ?? filePath,
-          ts: event.ts
-        };
-      }
-      if (type === "RunStarted") {
-        runStarted = { startedAt: event.startedAt };
-      }
-      if (type === "RunCompleted") {
-        runCompleted = {
-          passedTestCases: event.passedTestCases,
-          failedTestCases: event.failedTestCases,
-          totalTestCases: event.totalTestCases,
-          finishedAt: event.finishedAt
-        };
-      }
-      if (type === "RunFailed") {
-        runFailed = {
-          finishedAt: event.finishedAt,
-          errorMessage: event.errorMessage
-        };
-      }
-    } catch {
-    }
+
+// src/runner/name-pattern.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
   }
-  if (!runQueued) {
-    return null;
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
   }
-  const artifactPath = filePath;
-  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
-  const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
-  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
-  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
-    runId: runQueued.runId,
-    datasetId: runQueued.datasetId,
-    datasetName: runQueued.datasetName,
-    evaluatorIds: runQueued.evaluatorIds,
-    queuedAt: runQueued.ts ?? 0,
-    startedAt: runStarted?.startedAt,
-    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
-    totalTestCases: runQueued.totalTestCases,
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
-    status,
-    artifactPath,
-    errorMessage: runFailed?.errorMessage
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
   };
 }
-function aggregateTestCaseProgress(lines) {
-  let completedTestCases = 0;
-  const testCasePassedBy = /* @__PURE__ */ new Map();
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      if (event.type === "TestCaseProgress") {
-        const ev = event;
-        completedTestCases = ev.completedTestCases ?? completedTestCases;
-        const id = ev.testCaseId;
-        const current = testCasePassedBy.get(id);
-        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
-      }
-    } catch {
-    }
-  }
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const passed of testCasePassedBy.values()) {
-    if (passed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
   }
-  return { completedTestCases, passedTestCases, failedTestCases };
-}
-async function parseArtifactFile(artifactPath) {
-  try {
-    const content = await readFile(artifactPath, "utf8");
-    const lines = content.split("\n").filter((line) => line.trim().length > 0);
-    const results = [];
-    for (const line of lines) {
-      try {
-        const event = JSON.parse(line);
-        if (event.type === "TestCaseProgress") {
-          const ev = event;
-          results.push({
-            testCaseId: ev.testCaseId,
-            testCaseName: ev.testCaseName,
-            completedTestCases: ev.completedTestCases,
-            totalTestCases: ev.totalTestCases,
-            rerunIndex: ev.rerunIndex,
-            rerunTotal: ev.rerunTotal,
-            passed: ev.passed,
-            durationMs: ev.durationMs,
-            evaluatorScores: ev.evaluatorScores ?? []
-          });
-        }
-      } catch {
-      }
-    }
-    return results;
-  } catch {
-    return [];
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
   }
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
 async function appendJsonLine(artifactPath, payload) {
   await mkdir(dirname(artifactPath), { recursive: true });
@@ -1886,32 +1977,12 @@ function searchCollectedTestCases(all, query) {
 }
 
 // src/runner/api.ts
-function parseRegexLiteral(pattern) {
-  if (!pattern.startsWith("/")) {
-    return void 0;
-  }
-  const lastSlash = pattern.lastIndexOf("/");
-  if (lastSlash <= 0) {
-    return void 0;
-  }
-  return {
-    source: pattern.slice(1, lastSlash),
-    flags: pattern.slice(lastSlash + 1)
-  };
-}
-function createNameMatcher(pattern) {
-  const normalizedPattern = pattern.trim();
-  const regexLiteral = parseRegexLiteral(normalizedPattern);
-  if (regexLiteral) {
-    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
-    return (value) => regex.test(value);
+function normalizeRunRepetitions(value) {
+  const n = value ?? 1;
+  if (!Number.isInteger(n) || n < 1) {
+    throw new Error(`repetitions must be a positive integer, got ${String(value)}`);
   }
-  if (normalizedPattern.includes("*")) {
-    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-    const regex = new RegExp(`^${escaped}$`, "i");
-    return (value) => regex.test(value);
-  }
-  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+  return n;
 }
 function mergeRunnerOverrides(base, next) {
   if (!base) {
@@ -1942,15 +2013,12 @@ var EffectRunner = class {
     this.persistenceQueue = Effect.runSync(
       Queue.unbounded()
     );
-    this.snapshotsRef = Effect.runSync(
-      Ref.make(/* @__PURE__ */ new Map())
-    );
+    this.snapshotsRef = Effect.runSync(Ref.make(/* @__PURE__ */ new Map()));
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
-    this.schedulerFiber = Effect.runFork(
-      this.createSchedulerEffect()
-    );
+    this.runConfigsById = /* @__PURE__ */ new Map();
+    this.schedulerFiber = Effect.runFork(this.createSchedulerEffect());
     this.persistenceFiber = Effect.runFork(
       createPersistenceWorker(this.persistenceQueue)
     );
@@ -1990,6 +2058,137 @@ var EffectRunner = class {
       (item) => matcher(item.evaluator.getName() ?? "")
     );
   }
+  async collectRunConfigs() {
+    const runConfigs = await collectRunConfigsFromFiles(this.config.discovery);
+    this.runConfigsById.clear();
+    const byNameLower = /* @__PURE__ */ new Map();
+    for (const item of runConfigs) {
+      const id = item.runConfig.getName();
+      const lower = id.toLowerCase();
+      const prev = byNameLower.get(lower);
+      if (prev !== void 0 && prev.filePath !== item.filePath) {
+        throw new Error(
+          `Duplicate RunConfig name "${id}" (matches "${prev.runConfig.getName()}" case-insensitively): ${prev.filePath} and ${item.filePath}`
+        );
+      }
+      byNameLower.set(lower, item);
+      this.runConfigsById.set(id, item);
+    }
+    return runConfigs;
+  }
+  async resolveRunConfigByName(name) {
+    if (this.runConfigsById.size === 0) {
+      await this.collectRunConfigs();
+    }
+    const key = validateRunConfigName(name, `RunConfig "${name.trim()}"`);
+    const keyLower = key.toLowerCase();
+    const matches = Array.from(this.runConfigsById.values()).filter(
+      (item) => item.runConfig.getName().toLowerCase() === keyLower
+    );
+    if (matches.length === 0) {
+      return void 0;
+    }
+    if (matches.length > 1) {
+      throw new Error(
+        `Multiple RunConfigs named "${name}": ${matches.map((m) => m.filePath).join(", ")}`
+      );
+    }
+    return matches[0];
+  }
+  async expandRunConfigToJobs(collected) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const rcName = collected.runConfig.getName();
+    const jobs = [];
+    const runs = collected.runConfig.getRuns();
+    for (const [i, row] of runs.entries()) {
+      const dsCollected = Array.from(this.datasetsById.values()).find(
+        (d) => d.dataset === row.dataset
+      );
+      if (!dsCollected) {
+        throw new Error(
+          `RunConfig "${rcName}" run[${i}]: dataset "${row.dataset.getName()}" was not found among discovered dataset exports (import the same module instances the scanner loads).`
+        );
+      }
+      let evaluatorIds;
+      if ("evaluatorPattern" in row && typeof row.evaluatorPattern === "string") {
+        const matcher = createNameMatcher(row.evaluatorPattern);
+        const matched = Array.from(this.evaluatorsById.values()).filter(
+          (item) => matcher(item.evaluator.getName() ?? "")
+        );
+        if (matched.length === 0) {
+          throw new Error(
+            `RunConfig "${rcName}" run[${i}]: no evaluator matched pattern "${row.evaluatorPattern}"`
+          );
+        }
+        evaluatorIds = matched.map((item) => item.id);
+      } else {
+        const evaluators = row.evaluators;
+        evaluatorIds = [];
+        for (const ev of evaluators) {
+          const found = Array.from(this.evaluatorsById.values()).find(
+            (item) => item.evaluator === ev
+          );
+          if (!found) {
+            throw new Error(
+              `RunConfig "${rcName}" run[${i}]: evaluator "${getEvaluatorDisplayLabel(ev) ?? "unknown"}" was not found among discovered evaluator exports`
+            );
+          }
+          evaluatorIds.push(found.id);
+        }
+      }
+      const repetitions = "repetitions" in row && row.repetitions !== void 0 ? row.repetitions : 1;
+      jobs.push({
+        datasetId: dsCollected.id,
+        evaluatorIds,
+        runConfigName: rcName,
+        runConfigDisplayLabel: collected.runConfig.getDisplayLabel(),
+        runConfigTags: collected.runConfig.getTags(),
+        repetitions
+      });
+    }
+    return jobs;
+  }
+  async expandRunConfigNamesToJobs(names) {
+    const jobs = [];
+    for (const name of names) {
+      const collected = await this.resolveRunConfigByName(name);
+      if (!collected) {
+        const known = await this.collectRunConfigs();
+        const available = known.map((r) => r.runConfig.getName()).sort();
+        throw new Error(
+          available.length > 0 ? `RunConfig "${name}" not found. Available RunConfigs: ${available.join(", ")}` : `RunConfig "${name}" not found and no RunConfigs were discovered.`
+        );
+      }
+      jobs.push(...await this.expandRunConfigToJobs(collected));
+    }
+    return jobs;
+  }
+  async runDatasetJobsWithSharedConcurrency(request) {
+    const globalConcurrency = Math.max(1, request.globalConcurrency);
+    const sem = Effect.unsafeMakeSemaphore(globalConcurrency);
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const snapshots = [];
+    for (const job of request.jobs) {
+      snapshots.push(
+        await this.startDatasetRun({
+          datasetId: job.datasetId,
+          evaluatorIds: job.evaluatorIds,
+          triggerId,
+          maxConcurrency: this.config.maxConcurrency ?? 1,
+          globalEvaluationSemaphore: sem,
+          runConfigName: job.runConfigName,
+          runConfigTags: job.runConfigTags,
+          repetitions: job.repetitions
+        })
+      );
+    }
+    return snapshots;
+  }
   async searchTestCases(query) {
     const testCases = await collectTestCasesFromFiles(this.config.discovery);
     return searchCollectedTestCases(testCases, query);
@@ -2008,35 +2207,45 @@ var EffectRunner = class {
     );
   }
   async runDatasetWith(request) {
+    const runConfigName = validateRunConfigName(
+      request.runConfigName,
+      "runDatasetWith.runConfigName"
+    );
+    return this.startDatasetRun({
+      datasetId: request.datasetId,
+      evaluatorIds: request.evaluatorIds,
+      triggerId: request.triggerId,
+      maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
+      repetitions: request.repetitions,
+      runConfigName,
+      runConfigTags: request.runConfigTags
+    });
+  }
+  async startDatasetRun(params) {
     if (this.datasetsById.size === 0) {
       await this.collectDatasets();
     }
     if (this.evaluatorsById.size === 0) {
       await this.collectEvaluators();
     }
-    const dataset = this.datasetsById.get(request.datasetId);
+    const dataset = this.datasetsById.get(params.datasetId);
     if (!dataset) {
-      throw new Error(`Unknown dataset: ${request.datasetId}`);
+      throw new Error(`Unknown dataset: ${params.datasetId}`);
     }
-    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    const selectedEvaluators = params.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
     if (selectedEvaluators.length === 0) {
       throw new Error("No evaluators selected for run");
     }
-    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
-    const totalEvaluations = selectedTestCases.reduce(
-      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-      0
-    );
-    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const selectedTestCases = await this.collectDatasetTestCases(params.datasetId);
+    const repetitions = normalizeRunRepetitions(params.repetitions);
+    const totalEvaluations = selectedTestCases.length * repetitions;
+    const runConfigTags = [...params.runConfigTags ?? []];
+    const triggerId = params.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
-    const artifactPath = createArtifactPath(
-      this.config.artifactDirectory,
-      request.datasetId,
-      runId
-    );
+    const artifactPath = createArtifactPath(this.config.artifactDirectory, params.datasetId, runId);
     const snapshot = {
       runId,
-      datasetId: request.datasetId,
+      datasetId: params.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
@@ -2057,7 +2266,7 @@ var EffectRunner = class {
     const queuedEvent = {
       type: "RunQueued",
       runId,
-      datasetId: request.datasetId,
+      datasetId: params.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       totalTestCases: totalEvaluations,
@@ -2071,17 +2280,20 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
-    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
         triggerId,
-        datasetId: request.datasetId,
+        datasetId: params.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
         snapshot,
-        maxConcurrency
+        maxConcurrency: params.maxConcurrency,
+        globalEvaluationSemaphore: params.globalEvaluationSemaphore,
+        runConfigName: params.runConfigName,
+        runConfigTags,
+        repetitions
       })
     );
     return snapshot;
@@ -2097,9 +2309,9 @@ var EffectRunner = class {
     return Effect.runSync(Ref.get(this.snapshotsRef)).get(runId);
   }
   getAllRunSnapshots() {
-    return Array.from(
-      Effect.runSync(Ref.get(this.snapshotsRef)).values()
-    ).sort((a, b) => b.queuedAt - a.queuedAt);
+    return Array.from(Effect.runSync(Ref.get(this.snapshotsRef)).values()).sort(
+      (a, b) => b.queuedAt - a.queuedAt
+    );
   }
   async loadRunSnapshotsFromArtifacts() {
     return loadRunSnapshotsFromArtifacts(this.config);
@@ -2152,6 +2364,11 @@ var EffectRunner = class {
     );
   }
 };
+
+// src/runner/events.ts
+var PROGRAMMATIC_RUN_CONFIG = {
+  runConfigName: "programmatic"
+};
 var LEFT_PANE_WIDTH2 = 44;
 var MAX_RUNS_FOR_CHART = 12;
 var MAX_RUNS_FOR_TREND = 20;
@@ -2315,11 +2532,7 @@ function DatasetsView({
     ] })
   ] });
 }
-function RunsView({
-  state,
-  dataset,
-  selectedRun
-}) {
+function RunsView({ state, dataset, selectedRun }) {
   const runs = dataset?.runs ?? [];
   const rightFocused = state.focus === "right";
   return /* @__PURE__ */ jsxs(Fragment, { children: [
@@ -2335,10 +2548,10 @@ function RunsView({
       /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
         "Commit: ",
         selectedRun.meta.commit,
-        "  Branch: ",
+        " Branch: ",
         selectedRun.meta.branch,
+        " Seed:",
         " ",
-        "Seed: ",
         selectedRun.meta.seed
       ] }),
       /* @__PURE__ */ jsx(Text, { children: " " }),
@@ -2351,23 +2564,10 @@ function RunsView({
           format: (v) => `${v}%`
         }
       ),
-      /* @__PURE__ */ jsx(
-        TextBar,
-        {
-          label: "avg score",
-          value: Math.round(selectedRun.performance.avgScore * 100)
-        }
-      ),
+      /* @__PURE__ */ jsx(TextBar, { label: "avg score", value: Math.round(selectedRun.performance.avgScore * 100) }),
       /* @__PURE__ */ jsx(Text, { children: " " }),
       /* @__PURE__ */ jsx(SectionHeader, { children: "Dimensions" }),
-      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsx(
-        TextBar,
-        {
-          label: dimension.name,
-          value: dimension.score
-        },
-        dimension.name
-      )),
+      selectedRun.dimensions.map((dimension) => /* @__PURE__ */ jsx(TextBar, { label: dimension.name, value: dimension.score }, dimension.name)),
       /* @__PURE__ */ jsx(Text, { children: " " }),
       /* @__PURE__ */ jsx(SectionHeader, { children: "Latency trend" }),
       /* @__PURE__ */ jsx(
@@ -2470,15 +2670,7 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     ...dimensions.map((d) => /* @__PURE__ */ jsx(TextBar, { label: d.name, value: d.score }, `dim-${d.name}`)),
     /* @__PURE__ */ jsx(Text, { children: " " }, "sp2"),
     /* @__PURE__ */ jsx(SectionHeader, { children: "Checks (boolean)" }, "checks-h"),
-    ...checks.map((c) => /* @__PURE__ */ jsx(
-      CheckRow,
-      {
-        name: c.name,
-        passed: c.passed,
-        detail: c.detail
-      },
-      `chk-${c.name}`
-    )),
+    ...checks.map((c) => /* @__PURE__ */ jsx(CheckRow, { name: c.name, passed: c.passed, detail: c.detail }, `chk-${c.name}`)),
     /* @__PURE__ */ jsx(Text, { children: " " }, "sp3"),
     /* @__PURE__ */ jsx(SectionHeader, { children: "Performance" }, "perf-h"),
     /* @__PURE__ */ jsx(
@@ -2524,7 +2716,7 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp6"));
     rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Test cases" }, "tc-h"));
     for (const tc of testCases) {
-      const rerunPart = tc.rerunTotal != null && tc.rerunIndex != null ? ` (${tc.rerunIndex}/${tc.rerunTotal})` : "";
+      const repetitionPart = tc.repetitionCount != null && tc.repetitionCount > 1 && tc.repetitionIndex != null ? ` (${tc.repetitionIndex}/${tc.repetitionCount})` : "";
       rows.push(
         /* @__PURE__ */ jsxs(Text, { children: [
           /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
@@ -2536,13 +2728,13 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
           ] }),
           " ",
           tc.testCaseName,
-          rerunPart ? /* @__PURE__ */ jsx(Text, { color: "cyan", children: rerunPart }) : null,
+          repetitionPart ? /* @__PURE__ */ jsx(Text, { color: "cyan", children: repetitionPart }) : null,
           /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
             " (",
             tc.durationMs,
             "ms)"
           ] })
-        ] }, `tc-${tc.testCaseId}-${tc.rerunIndex ?? 0}`)
+        ] }, `tc-${tc.testCaseId}-${tc.repetitionId ?? "x"}-${tc.repetitionIndex ?? 0}`)
       );
       for (const item of tc.evaluatorScores) {
         const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;
@@ -2595,17 +2787,10 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
           }
         } else {
           rows.push(
-            /* @__PURE__ */ jsxs(
-              Text,
-              {
-                color: "gray",
-                children: [
-                  "      ",
-                  "n/a"
-                ]
-              },
-              `tc-${tc.testCaseId}-${item.evaluatorId}-n/a`
-            )
+            /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+              "      ",
+              "n/a"
+            ] }, `tc-${tc.testCaseId}-${item.evaluatorId}-n/a`)
           );
         }
         if (!item.passed && item.logs && item.logs.length > 0) {
@@ -2663,7 +2848,7 @@ function RunDetailsView({
   const runs = dataset?.runs ?? [];
   const rightFocused = state.focus === "right";
   const [testCases, setTestCases] = useState([]);
-  const evaluatorNameById = React2.useMemo(
+  const evaluatorNameById = React.useMemo(
     () => new Map(evaluators.map((e) => [e.id, e.name])),
     [evaluators]
   );
@@ -2686,7 +2871,7 @@ function RunDetailsView({
   const visible = rows.slice(offset, offset + DETAILS_PAGE_SIZE);
   return /* @__PURE__ */ jsxs(Fragment, { children: [
     /* @__PURE__ */ jsx(RunsSidebar, { state, dataset, runs }),
-    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React2.Fragment, { children: row }, i)) }) })
+    /* @__PURE__ */ jsx(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: /* @__PURE__ */ jsx(Box, { flexDirection: "column", children: visible.map((row, i) => /* @__PURE__ */ jsx(React.Fragment, { children: row }, i)) }) })
   ] });
 }
 var LEFT_PANE_WIDTH3 = 44;
@@ -2709,19 +2894,11 @@ function NewEvaluationView({
       visibleEvaluators.map((evaluator, index) => {
         const selected = index === state.evaluatorMenuIndex;
         const inSelection = state.selectedEvaluatorIds.includes(evaluator.id);
-        return /* @__PURE__ */ jsxs(
-          Text,
-          {
-            color: selected ? "cyan" : "gray",
-            bold: selected,
-            children: [
-              selected ? "\u25B8 " : "  ",
-              inSelection ? "[x] " : "[ ] ",
-              evaluator.name
-            ]
-          },
-          evaluator.id
-        );
+        return /* @__PURE__ */ jsxs(Text, { color: selected ? "cyan" : "gray", bold: selected, children: [
+          selected ? "\u25B8 " : "  ",
+          inSelection ? "[x] " : "[ ] ",
+          evaluator.name
+        ] }, evaluator.id);
       })
     ] }),
     /* @__PURE__ */ jsxs(Pane, { flexGrow: 1, marginLeft: 1, focused: rightFocused, children: [
@@ -2753,26 +2930,16 @@ function clampCursor(state, filteredDatasetsLength, selectedRunCount) {
     ...state,
     datasetMenuIndex: Math.max(0, Math.min(state.datasetMenuIndex, datasetMax)),
     runMenuIndex: Math.max(0, Math.min(state.runMenuIndex, runMax)),
-    evaluatorMenuIndex: Math.max(
-      0,
-      Math.min(state.evaluatorMenuIndex, evaluatorMax)
-    )
+    evaluatorMenuIndex: Math.max(0, Math.min(state.evaluatorMenuIndex, evaluatorMax))
   };
 }
-function EvalsCliApp({
-  data,
-  args,
-  runner
-}) {
+function EvalsCliApp({ data, args, runner }) {
   const { exit } = useApp();
   const { width: stdoutWidth, height: stdoutHeight } = useScreenSize();
   const [liveData, setLiveData] = useState(data);
   const [runtimeMessage, setRuntimeMessage] = useState();
   const overviewRowCountRef = useRef(0);
-  const [state, dispatch] = useReducer(
-    reduceCliState,
-    createInitialState(data, args)
-  );
+  const [state, dispatch] = useReducer(reduceCliState, createInitialState(data, args));
   useEffect(() => {
     setLiveData(data);
   }, [data]);
@@ -2804,14 +2971,8 @@ function EvalsCliApp({
     filteredDatasets.length,
     getDatasetByMenuIndex(filteredDatasets, state.datasetMenuIndex)?.runs.length ?? 0
   );
-  const selectedDataset = getDatasetByMenuIndex(
-    filteredDatasets,
-    clampedState.datasetMenuIndex
-  );
-  const selectedRun = getRunByMenuIndex(
-    selectedDataset,
-    clampedState.runMenuIndex
-  );
+  const selectedDataset = getDatasetByMenuIndex(filteredDatasets, clampedState.datasetMenuIndex);
+  const selectedRun = getRunByMenuIndex(selectedDataset, clampedState.runMenuIndex);
   const visibleEvaluators = liveData.evaluators.filter(
     (evaluator) => evaluator.name.toLowerCase().includes(clampedState.searchQuery.toLowerCase())
   );
@@ -2899,15 +3060,14 @@ function EvalsCliApp({
       }
       void runner.runDatasetWith({
         datasetId: selectedDataset.id,
-        evaluatorIds: clampedState.selectedEvaluatorIds
+        evaluatorIds: clampedState.selectedEvaluatorIds,
+        ...PROGRAMMATIC_RUN_CONFIG
       }).then((snapshot) => {
         setRuntimeMessage(
           `Started ${snapshot.runId} on ${selectedDataset.name} (${snapshot.totalTestCases} cases).`
         );
       }).catch((error) => {
-        setRuntimeMessage(
-          error instanceof Error ? error.message : "Failed to start evaluation."
-        );
+        setRuntimeMessage(error instanceof Error ? error.message : "Failed to start evaluation.");
       });
     }
   });
@@ -2934,14 +3094,7 @@ function EvalsCliApp({
       );
     }
     if (clampedState.level === "runs") {
-      return /* @__PURE__ */ jsx(
-        RunsView,
-        {
-          state: clampedState,
-          dataset: selectedDataset,
-          selectedRun
-        }
-      );
+      return /* @__PURE__ */ jsx(RunsView, { state: clampedState, dataset: selectedDataset, selectedRun });
     }
     return /* @__PURE__ */ jsx(
       RunDetailsView,
@@ -2953,82 +3106,44 @@ function EvalsCliApp({
       }
     );
   };
-  return /* @__PURE__ */ jsxs(
-    Box,
-    {
-      flexDirection: "column",
-      flexGrow: 1,
-      width: stdoutWidth,
-      height: stdoutHeight,
-      children: [
-        /* @__PURE__ */ jsx(
-          Box,
-          {
-            borderStyle: "round",
-            borderColor: "cyan",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: /* @__PURE__ */ jsx(Text, { children: getBreadcrumbText(
-              clampedState,
-              selectedDataset?.name,
-              selectedRun?.label
-            ) })
-          }
-        ),
-        clampedState.startupWarnings.length > 0 && /* @__PURE__ */ jsxs(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "yellow",
-            paddingX: 1,
-            flexDirection: "column",
-            width: stdoutWidth,
-            children: [
-              /* @__PURE__ */ jsx(Text, { color: "yellow", children: "Startup warnings:" }),
-              clampedState.startupWarnings.map((warning, index) => /* @__PURE__ */ jsx(Text, { children: warning }, `${warning}-${index}`))
-            ]
-          }
-        ),
-        clampedState.searchMode && /* @__PURE__ */ jsxs(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "magenta",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: [
-              /* @__PURE__ */ jsx(Text, { color: "magenta", bold: true, children: "Search: " }),
-              /* @__PURE__ */ jsx(Text, { color: "white", children: clampedState.searchQuery })
-            ]
-          }
-        ),
-        runtimeMessage && /* @__PURE__ */ jsx(
-          Box,
-          {
-            marginTop: 1,
-            borderStyle: "round",
-            borderColor: "blue",
-            paddingX: 1,
-            width: stdoutWidth,
-            children: /* @__PURE__ */ jsx(Text, { color: "blue", children: runtimeMessage })
-          }
-        ),
-        /* @__PURE__ */ jsx(
-          Box,
-          {
-            marginTop: 1,
-            flexGrow: 1,
-            width: stdoutWidth,
-            flexDirection: "row",
-            children: renderContent()
-          }
-        ),
-        /* @__PURE__ */ jsx(Box, { marginTop: 1, paddingX: 1, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: getFooterText(clampedState) }) })
-      ]
-    }
-  );
+  return /* @__PURE__ */ jsxs(Box, { flexDirection: "column", flexGrow: 1, width: stdoutWidth, height: stdoutHeight, children: [
+    /* @__PURE__ */ jsx(Box, { borderStyle: "round", borderColor: "cyan", paddingX: 1, width: stdoutWidth, children: /* @__PURE__ */ jsx(Text, { children: getBreadcrumbText(clampedState, selectedDataset?.name, selectedRun?.label) }) }),
+    clampedState.startupWarnings.length > 0 && /* @__PURE__ */ jsxs(
+      Box,
+      {
+        marginTop: 1,
+        borderStyle: "round",
+        borderColor: "yellow",
+        paddingX: 1,
+        flexDirection: "column",
+        width: stdoutWidth,
+        children: [
+          /* @__PURE__ */ jsx(Text, { color: "yellow", children: "Startup warnings:" }),
+          clampedState.startupWarnings.map((warning, index) => /* @__PURE__ */ jsx(Text, { children: warning }, `${warning}-${index}`))
+        ]
+      }
+    ),
+    clampedState.searchMode && /* @__PURE__ */ jsxs(
+      Box,
+      {
+        marginTop: 1,
+        borderStyle: "round",
+        borderColor: "magenta",
+        paddingX: 1,
+        width: stdoutWidth,
+        children: [
+          /* @__PURE__ */ jsxs(Text, { color: "magenta", bold: true, children: [
+            "Search:",
+            " "
+          ] }),
+          /* @__PURE__ */ jsx(Text, { color: "white", children: clampedState.searchQuery })
+        ]
+      }
+    ),
+    runtimeMessage && /* @__PURE__ */ jsx(Box, { marginTop: 1, borderStyle: "round", borderColor: "blue", paddingX: 1, width: stdoutWidth, children: /* @__PURE__ */ jsx(Text, { color: "blue", children: runtimeMessage }) }),
+    /* @__PURE__ */ jsx(Box, { marginTop: 1, flexGrow: 1, width: stdoutWidth, flexDirection: "row", children: renderContent() }),
+    /* @__PURE__ */ jsx(Box, { marginTop: 1, paddingX: 1, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: getFooterText(clampedState) }) })
+  ] });
 }
 async function main() {
   const args = parseStartupArgs(process.argv.slice(2));
@@ -3040,9 +3155,7 @@ async function main() {
   process.on("SIGTERM", () => {
     void runner.shutdown().finally(() => process.exit(0));
   });
-  withFullScreen(
-    /* @__PURE__ */ jsx(EvalsCliApp, { data, args, runner })
-  ).start();
+  withFullScreen(/* @__PURE__ */ jsx(EvalsCliApp, { data, args, runner })).start();
 }
 void main();
 //# sourceMappingURL=out.js.map