@m4trix/evals 0.25.1 → 0.27.0

package/dist/index.js

@@ -1,14 +1,257 @@
-import { Effect, PubSub, Queue, Ref, Fiber } from 'effect';
+import { Schema, Effect, PubSub, Queue, Ref, Fiber, Either, ParseResult } from 'effect';
 export { Schema as S } from 'effect';
 import { diffLines } from 'diff';
 import stringify from 'fast-json-stable-stringify';
 import { randomUUID } from 'crypto';
+import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
+import { resolve as resolve$1, join, relative, dirname } from 'path';
 import { existsSync } from 'fs';
-import { resolve as resolve$1, relative, join, dirname } from 'path';
 import * as jitiModule from 'jiti';
-import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
 
+// src/index.ts
+var ENTITY_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
+function makeEntityIdSchema(brand, label) {
+  return Schema.String.pipe(
+    Schema.trimmed(),
+    Schema.minLength(1, {
+      message: () => `${label} must be non-empty.`
+    }),
+    Schema.pattern(ENTITY_ID_PATTERN, {
+      message: () => `${label} may only contain letters, digits, underscores, and hyphens (no spaces). Examples: "my-nightly", "my_nightly", "myNightly".`
+    }),
+    Schema.brand(brand)
+  );
+}
+var RunConfigNameSchema = makeEntityIdSchema("RunConfigName", "RunConfig name");
+var EvaluatorNameSchema = makeEntityIdSchema("EvaluatorName", "Evaluator name");
+var TestCaseNameSchema = makeEntityIdSchema("TestCaseName", "Test case name");
+var DatasetNameSchema = makeEntityIdSchema("DatasetName", "Dataset name");
+function validateWithSchema(schema, raw, context) {
+  const trimmed = raw.trim();
+  const decode = Schema.decodeUnknownEither(
+    schema
+  );
+  const result = decode(trimmed);
+  if (Either.isLeft(result)) {
+    throw new Error(`${context}: ${ParseResult.TreeFormatter.formatErrorSync(result.left)}`);
+  }
+  return result.right;
+}
+function validateRunConfigName(raw, context) {
+  return validateWithSchema(RunConfigNameSchema, raw, context);
+}
+function validateEvaluatorName(raw, context) {
+  return validateWithSchema(EvaluatorNameSchema, raw, context);
+}
+function validateTestCaseName(raw, context) {
+  return validateWithSchema(TestCaseNameSchema, raw, context);
+}
+function validateDatasetName(raw, context) {
+  return validateWithSchema(DatasetNameSchema, raw, context);
+}
+function normalizeOptionalDisplayName(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  const t = raw.trim();
+  return t.length === 0 ? void 0 : t;
+}
+
+// src/evals/dataset.ts
+function matchesAny(value, matchers) {
+  return matchers.some(
+    (matcher) => typeof matcher === "string" ? value === matcher : matcher.test(value)
+  );
+}
+function matchesAnyPath(filePath, matchers) {
+  return matchers.some((matcher) => {
+    if (typeof matcher === "string") {
+      return simpleGlobMatch(matcher, filePath);
+    }
+    return matcher.test(filePath);
+  });
+}
+function simpleGlobMatch(pattern, value) {
+  const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\?/g, "[^/]").replace(/\*\*\//g, "(?:.*/)?").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*");
+  return new RegExp(`^${escaped}$`).test(value);
+}
+var Dataset = class _Dataset {
+  constructor(config) {
+    this._config = config;
+  }
+  static define(config) {
+    const name = validateDatasetName(config.name, "Dataset.define");
+    const displayName = normalizeOptionalDisplayName(config.displayName);
+    return new _Dataset({
+      name,
+      displayName,
+      includedTags: config.includedTags ?? [],
+      excludedTags: config.excludedTags ?? [],
+      includedPaths: config.includedPaths ?? [],
+      excludedPaths: config.excludedPaths ?? []
+    });
+  }
+  /** Canonical dataset id (same rules as `RunConfig` / `TestCase` `name`). */
+  getName() {
+    return this._config.name;
+  }
+  getDisplayName() {
+    return this._config.displayName;
+  }
+  /** Label for CLI/TUI and evaluator `meta.datasetName`: {@link getDisplayName} if set, otherwise {@link getName}. */
+  getDisplayLabel() {
+    return this._config.displayName ?? this._config.name;
+  }
+  getIncludedTags() {
+    return this._config.includedTags;
+  }
+  getExcludedTags() {
+    return this._config.excludedTags;
+  }
+  getIncludedPaths() {
+    return this._config.includedPaths;
+  }
+  getExcludedPaths() {
+    return this._config.excludedPaths;
+  }
+  matchesTestCase(testCase, filePath) {
+    const tags = testCase.getTags();
+    if (this._config.excludedTags.length > 0) {
+      if (tags.some((tag) => matchesAny(tag, this._config.excludedTags))) {
+        return false;
+      }
+    }
+    if (this._config.excludedPaths.length > 0) {
+      if (matchesAnyPath(filePath, this._config.excludedPaths)) {
+        return false;
+      }
+    }
+    const tagMatch = this._config.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, this._config.includedTags));
+    const pathMatch = this._config.includedPaths.length === 0 || matchesAnyPath(filePath, this._config.includedPaths);
+    return tagMatch && pathMatch;
+  }
+};
+function getDatasetDisplayLabel(dataset) {
+  if (typeof dataset.getDisplayLabel === "function") {
+    return dataset.getDisplayLabel();
+  }
+  return typeof dataset.getName === "function" ? dataset.getName() : "";
+}
+
+// src/evals/evaluator.ts
+var Evaluator = class _Evaluator {
+  constructor(config) {
+    this._config = config;
+  }
+  getState() {
+    return {
+      name: this._config.name,
+      displayName: this._config.displayName,
+      tags: this._config.tags,
+      inputSchema: this._config.inputSchema,
+      outputSchema: this._config.outputSchema,
+      scoreSchema: this._config.scoreSchema,
+      middlewares: this._config.middlewares,
+      evaluateFn: this._config.evaluateFn,
+      passThreshold: this._config.passThreshold,
+      passCriterion: this._config.passCriterion
+    };
+  }
+  static use(middleware) {
+    return new _Evaluator({
+      middlewares: [middleware],
+      tags: []
+    });
+  }
+  use(middleware) {
+    const state = this.getState();
+    return new _Evaluator({
+      ...state,
+      middlewares: [...state.middlewares, middleware]
+    });
+  }
+  define(config) {
+    const { middlewares } = this.getState();
+    const name = validateEvaluatorName(config.name, "Evaluator.define");
+    const displayName = normalizeOptionalDisplayName(config.displayName);
+    const tags = config.tags !== void 0 ? [...config.tags] : [];
+    return new _Evaluator({
+      name,
+      displayName,
+      tags,
+      inputSchema: config.inputSchema,
+      outputSchema: config.outputSchema,
+      scoreSchema: config.scoreSchema,
+      middlewares,
+      passThreshold: config.passThreshold,
+      passCriterion: config.passCriterion
+    });
+  }
+  evaluate(fn) {
+    return new _Evaluator({
+      ...this.getState(),
+      evaluateFn: fn
+    });
+  }
+  /** Canonical evaluator id when defined; otherwise undefined (middleware-only chain). */
+  getName() {
+    return this._config.name;
+  }
+  getDisplayName() {
+    return this._config.displayName;
+  }
+  /** Label for CLI/TUI: {@link getDisplayName} if set, otherwise {@link getName}. Undefined if not yet defined. */
+  getDisplayLabel() {
+    const id = this._config.name;
+    if (id === void 0) {
+      return void 0;
+    }
+    return this._config.displayName ?? id;
+  }
+  /** Tags from `Evaluator.define({ tags })`; empty until defined. */
+  getTags() {
+    return [...this._config.tags];
+  }
+  getInputSchema() {
+    return this._config.inputSchema;
+  }
+  getOutputSchema() {
+    return this._config.outputSchema;
+  }
+  getScoreSchema() {
+    return this._config.scoreSchema;
+  }
+  getMiddlewares() {
+    return this._config.middlewares;
+  }
+  getEvaluateFn() {
+    return this._config.evaluateFn;
+  }
+  getPassThreshold() {
+    return this._config.passThreshold;
+  }
+  getPassCriterion() {
+    return this._config.passCriterion;
+  }
+  async resolveContext() {
+    const parts = await Promise.all(this._config.middlewares.map((mw) => mw.resolve()));
+    return Object.assign({}, ...parts);
+  }
+};
+function getEvaluatorDisplayLabel(evaluator) {
+  if (typeof evaluator.getDisplayLabel === "function") {
+    const label = evaluator.getDisplayLabel();
+    if (label !== void 0) {
+      return label;
+    }
+  }
+  return typeof evaluator.getName === "function" ? evaluator.getName() : void 0;
+}
+function getEvaluatorTagList(evaluator) {
+  return typeof evaluator.getTags === "function" ? [...evaluator.getTags()] : [];
+}
+
 // src/cli/data.mock.json
 var data_mock_default = {
   datasets: [
@@ -255,7 +498,7 @@ function toEvalDataset(item, snapshots) {
   const runs = snapshots.filter((snapshot) => snapshot.datasetId === item.id).sort((a, b) => b.queuedAt - a.queuedAt).map(toEvalRun);
   return {
     id: item.id,
-    name: item.dataset.getName(),
+    name: getDatasetDisplayLabel(item.dataset),
     overview: `Discovered from ${item.filePath}`,
     runs
   };
@@ -263,7 +506,7 @@ function toEvalDataset(item, snapshots) {
 function toEvaluatorOption(item) {
   return {
     id: item.id,
-    name: item.evaluator.getName() ?? toSlug(item.id),
+    name: getEvaluatorDisplayLabel(item.evaluator) ?? toSlug(item.id),
     configPreview: `Source: ${item.filePath}`
   };
 }
@@ -308,196 +551,149 @@ function parseStartupArgs(argv) {
   }
   return args;
 }
-
-// src/evals/test-case.ts
-function resolve(value) {
-  return typeof value === "function" ? value() : value;
-}
-var TestCase = class _TestCase {
-  constructor(config) {
-    this._config = config;
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify(preprocessForDiff(a, options));
+      const bStr = stringify(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
   }
-  static describe(config) {
-    const reruns = config.reruns ?? 1;
-    if (reruns < 1 || !Number.isInteger(reruns)) {
-      throw new Error(`TestCase reruns must be a positive integer, got ${reruns}`);
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
     }
-    return new _TestCase({
-      name: config.name,
-      tags: config.tags,
-      reruns,
-      inputSchema: config.inputSchema,
-      input: config.input,
-      outputSchema: config.outputSchema,
-      output: config.output
-    });
+    return filtered;
   }
-  getReruns() {
-    return this._config.reruns;
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
   }
-  getName() {
-    return this._config.name;
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
   }
-  getTags() {
-    return this._config.tags;
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
   }
-  getInputSchema() {
-    return this._config.inputSchema;
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
   }
-  getInput() {
-    return resolve(this._config.input);
+  return lines.join("\n");
+}
+function createDiffString(expected, actual, diffOptions) {
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(extractKeys(expectedProcessed), null, 2);
+    const actualKeys = JSON.stringify(extractKeys(actualProcessed), null, 2);
+    const parts2 = diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
   }
-  getOutputSchema() {
-    return this._config.outputSchema;
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
   }
-  getOutput() {
-    if (this._config.output === void 0) {
-      return void 0;
-    }
-    return resolve(this._config.output);
-  }
-};
-
-// src/evals/evaluator.ts
-var Evaluator = class _Evaluator {
-  constructor(config) {
-    this._config = config;
-  }
-  getState() {
-    return {
-      name: this._config.name,
-      inputSchema: this._config.inputSchema,
-      outputSchema: this._config.outputSchema,
-      scoreSchema: this._config.scoreSchema,
-      middlewares: this._config.middlewares,
-      evaluateFn: this._config.evaluateFn,
-      passThreshold: this._config.passThreshold,
-      passCriterion: this._config.passCriterion
-    };
-  }
-  static use(middleware) {
-    return new _Evaluator({
-      middlewares: [middleware]
-    });
-  }
-  use(middleware) {
-    const state = this.getState();
-    return new _Evaluator({
-      ...state,
-      middlewares: [...state.middlewares, middleware]
-    });
-  }
-  define(config) {
-    const { middlewares } = this.getState();
-    return new _Evaluator({
-      name: config.name,
-      inputSchema: config.inputSchema,
-      outputSchema: config.outputSchema,
-      scoreSchema: config.scoreSchema,
-      middlewares,
-      passThreshold: config.passThreshold,
-      passCriterion: config.passCriterion
-    });
-  }
-  evaluate(fn) {
-    return new _Evaluator({
-      ...this.getState(),
-      evaluateFn: fn
-    });
-  }
-  getName() {
-    return this._config.name;
-  }
-  getInputSchema() {
-    return this._config.inputSchema;
-  }
-  getOutputSchema() {
-    return this._config.outputSchema;
-  }
-  getScoreSchema() {
-    return this._config.scoreSchema;
-  }
-  getMiddlewares() {
-    return this._config.middlewares;
-  }
-  getEvaluateFn() {
-    return this._config.evaluateFn;
+  const parts = diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter((p) => p.added === true);
+    return formatDiffParts(filtered);
   }
-  getPassThreshold() {
-    return this._config.passThreshold;
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
   }
-  getPassCriterion() {
-    return this._config.passCriterion;
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
   }
-  async resolveContext() {
-    const parts = await Promise.all(this._config.middlewares.map((mw) => mw.resolve()));
-    return Object.assign({}, ...parts);
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
   }
-};
-
-// src/evals/dataset.ts
-function matchesAny(value, matchers) {
-  return matchers.some(
-    (matcher) => typeof matcher === "string" ? value === matcher : matcher.test(value)
-  );
+  return result;
 }
-function matchesAnyPath(filePath, matchers) {
-  return matchers.some((matcher) => {
-    if (typeof matcher === "string") {
-      return simpleGlobMatch(matcher, filePath);
+function formatLogMessage(msg) {
+  if (typeof msg === "string")
+    return msg;
+  if (msg instanceof Error)
+    return msg.stack ?? msg.message;
+  try {
+    if (msg !== null && typeof msg === "object") {
+      return JSON.stringify(msg, null, 2);
     }
-    return matcher.test(filePath);
-  });
+    return String(msg);
+  } catch {
+    return String(msg);
+  }
 }
-function simpleGlobMatch(pattern, value) {
-  const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\?/g, "[^/]").replace(/\*\*\//g, "(?:.*/)?").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*");
-  return new RegExp(`^${escaped}$`).test(value);
+function createLogEntry(message, options) {
+  return {
+    type: "log",
+    label: options?.label,
+    message: formatLogMessage(message)
+  };
 }
-var Dataset = class _Dataset {
-  constructor(config) {
-    this._config = config;
-  }
-  static define(config) {
-    return new _Dataset({
-      name: config.name,
-      includedTags: config.includedTags ?? [],
-      excludedTags: config.excludedTags ?? [],
-      includedPaths: config.includedPaths ?? [],
-      excludedPaths: config.excludedPaths ?? []
-    });
-  }
-  getName() {
-    return this._config.name;
-  }
-  getIncludedTags() {
-    return this._config.includedTags;
-  }
-  getExcludedTags() {
-    return this._config.excludedTags;
-  }
-  getIncludedPaths() {
-    return this._config.includedPaths;
-  }
-  getExcludedPaths() {
-    return this._config.excludedPaths;
-  }
-  matchesTestCase(testCase, filePath) {
-    const tags = testCase.getTags();
-    if (this._config.excludedTags.length > 0) {
-      if (tags.some((tag) => matchesAny(tag, this._config.excludedTags))) {
-        return false;
+function getLogLines(entry) {
+  return entry.message.split("\n");
+}
+function createDiffLogEntry(expected, actual, options) {
+  const { label, ...diffOpts } = options ?? {};
+  const diff = createDiffString(expected, actual, diffOpts);
+  return {
+    type: "diff",
+    label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function printJsonDiff(expected, actual, options = {}) {
+  const { color = true, ...diffOpts } = options;
+  const diff = createDiffString(expected, actual, diffOpts);
+  if (color) {
+    const lines = diff.split("\n").map((line) => {
+      const trimmed = line.trimStart();
+      if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+        return `\x1B[31m${line}\x1B[0m`;
       }
-    }
-    if (this._config.excludedPaths.length > 0) {
-      if (matchesAnyPath(filePath, this._config.excludedPaths)) {
-        return false;
+      if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+        return `\x1B[32m${line}\x1B[0m`;
       }
-    }
-    const tagMatch = this._config.includedTags.length === 0 || tags.some((tag) => matchesAny(tag, this._config.includedTags));
-    const pathMatch = this._config.includedPaths.length === 0 || matchesAnyPath(filePath, this._config.includedPaths);
-    return tagMatch && pathMatch;
+      return line;
+    });
+    const colored = lines.join("\n");
+    console.log(colored || "(no differences)");
+    return colored;
   }
-};
+  console.log(diff || "(no differences)");
+  return diff;
+}
 
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
@@ -522,6 +718,113 @@ function getMetricById(id) {
   return registry.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
+// src/evals/metrics/standard.ts
+var tokenCountMetric = Metric.of({
+  id: "token-count",
+  name: "Tokens",
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
+    const input = data.input ?? 0;
+    const output = data.output ?? 0;
+    const inputCached = data.inputCached ?? 0;
+    const outputCached = data.outputCached ?? 0;
+    const cached = inputCached + outputCached;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
+  }
+});
+var latencyMetric = Metric.of({
+  id: "latency",
+  name: "Latency",
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
+});
+
+// src/evals/run-config.ts
+function validateRow(row, index) {
+  const hasEvaluators = "evaluators" in row && row.evaluators !== void 0 && row.evaluators !== void 0;
+  const hasPattern = "evaluatorPattern" in row && typeof row.evaluatorPattern === "string" && row.evaluatorPattern.trim().length > 0;
+  if (hasEvaluators && hasPattern) {
+    throw new Error(`RunConfig run[${index}] must not set both evaluators and evaluatorPattern`);
+  }
+  if (!hasEvaluators && !hasPattern) {
+    throw new Error(`RunConfig run[${index}] must set either evaluators or evaluatorPattern`);
+  }
+  if (hasEvaluators && row.evaluators.length === 0) {
+    throw new Error(`RunConfig run[${index}]: evaluators must be non-empty`);
+  }
+  const rawRep = "repetitions" in row ? row.repetitions : void 0;
+  const repetitions = rawRep ?? 1;
+  if (!Number.isInteger(repetitions) || repetitions < 1) {
+    throw new Error(
+      `RunConfig run[${index}]: repetitions must be a positive integer, got ${String(rawRep)}`
+    );
+  }
+}
+var RunConfig = class _RunConfig {
+  constructor(name, displayName, tags, runs) {
+    this._name = name;
+    this._displayName = displayName;
+    this._tags = tags;
+    this._runs = runs;
+  }
+  static define(config) {
+    if (config.runs.length === 0) {
+      throw new Error("RunConfig runs must be non-empty");
+    }
+    config.runs.forEach(validateRow);
+    const name = validateRunConfigName(config.name, "RunConfig.define");
+    const displayName = normalizeOptionalDisplayName(config.displayName);
+    const tags = config.tags !== void 0 ? [...config.tags] : [];
+    return new _RunConfig(name, displayName, tags, config.runs);
+  }
+  /** Canonical id (branded {@link RunConfigName} at runtime; typed as `string` for ergonomics). */
+  getName() {
+    return this._name;
+  }
+  /** Optional unrestricted display label. */
+  getDisplayName() {
+    return this._displayName;
+  }
+  /** Label for CLI/TUI: {@link getDisplayName} if set, otherwise {@link getName}. */
+  getDisplayLabel() {
+    return this._displayName ?? this._name;
+  }
+  /** Tags from `RunConfig.define({ tags })`; surfaced as `runConfigTags` on evaluator callbacks. */
+  getTags() {
+    return [...this._tags];
+  }
+  getRuns() {
+    return this._runs;
+  }
+};
+
 // src/evals/score.ts
 var registry2 = /* @__PURE__ */ new Map();
 function formatScoreData(def, data, options) {
@@ -612,71 +915,23 @@ var Score = {
       aggregateValues: config.aggregateValues,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
-        return {
-          id: config.id,
-          data,
-          ...passed !== void 0 && { passed },
-          ...options?.name !== void 0 && { name: options.name },
-          def
-          // Attach def so rendering/aggregation works without registry lookup
-        };
-      }
-    };
-    registry2.set(config.id, def);
-    return def;
-  }
-};
-function getScoreById(id) {
-  return registry2.get(id);
-}
-
-// src/evals/aggregators.ts
-function aggregateTokenCountSum(values) {
-  const initial = {
-    input: 0,
-    output: 0,
-    inputCached: 0,
-    outputCached: 0
-  };
-  return values.reduce(
-    (acc, v) => ({
-      input: acc.input + (v.input ?? 0),
-      output: acc.output + (v.output ?? 0),
-      inputCached: acc.inputCached + (v.inputCached ?? 0),
-      outputCached: acc.outputCached + (v.outputCached ?? 0)
-    }),
-    initial
-  );
-}
-function aggregateLatencyAverage(values) {
-  if (values.length === 0) {
-    return { ms: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.ms, 0);
-  return { ms: sum / values.length };
-}
-
-// src/evals/metrics/standard.ts
-var tokenCountMetric = Metric.of({
-  id: "token-count",
-  name: "Tokens",
-  aggregate: aggregateTokenCountSum,
-  format: (data, options) => {
-    const input = data.input ?? 0;
-    const output = data.output ?? 0;
-    const inputCached = data.inputCached ?? 0;
-    const outputCached = data.outputCached ?? 0;
-    const cached = inputCached + outputCached;
-    const base = `in:${input} out:${output} cached:${cached}`;
-    return options?.isAggregated ? `Total: ${base}` : base;
+        return {
+          id: config.id,
+          data,
+          ...passed !== void 0 && { passed },
+          ...options?.name !== void 0 && { name: options.name },
+          def
+          // Attach def so rendering/aggregation works without registry lookup
+        };
+      }
+    };
+    registry2.set(config.id, def);
+    return def;
   }
-});
-var latencyMetric = Metric.of({
-  id: "latency",
-  name: "Latency",
-  aggregate: aggregateLatencyAverage,
-  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
-});
+};
+function getScoreById(id) {
+  return registry2.get(id);
+}
 
 // src/evals/scores/standard.ts
 var percentScore = Score.of({
@@ -709,148 +964,197 @@ var binaryScore = Score.of({
   },
   aggregateValues: Score.aggregate.all
 });
-function preprocessForDiff(value, options) {
-  if (options?.sort && Array.isArray(value)) {
-    return [...value].sort((a, b) => {
-      const aStr = stringify(preprocessForDiff(a, options));
-      const bStr = stringify(preprocessForDiff(b, options));
-      return aStr.localeCompare(bStr);
-    }).map((item) => preprocessForDiff(item, options));
-  }
-  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
-    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
-    const filtered = {};
-    for (const [k, v] of Object.entries(value)) {
-      if (!keys.includes(k)) {
-        filtered[k] = preprocessForDiff(v, options);
-      }
+
+// src/evals/tag-set.ts
+var TagSet = class {
+  constructor() {
+  }
+  static define(tags) {
+    const out = {};
+    for (const tag of tags) {
+      out[tag] = tag;
     }
-    return filtered;
+    return out;
   }
-  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
-    const result = {};
-    for (const [k, v] of Object.entries(value)) {
-      result[k] = preprocessForDiff(v, options);
-    }
-    return result;
+};
+
+// src/evals/test-case.ts
+function resolve(value) {
+  return typeof value === "function" ? value() : value;
+}
+var TestCase = class _TestCase {
+  constructor(config) {
+    this._config = config;
   }
-  if (typeof value === "number" && options?.precision !== void 0) {
-    return Number(value.toFixed(options.precision));
+  static describe(config) {
+    const name = validateTestCaseName(config.name, "TestCase.describe");
+    const displayName = normalizeOptionalDisplayName(config.displayName);
+    return new _TestCase({
+      name,
+      displayName,
+      tags: config.tags,
+      inputSchema: config.inputSchema,
+      input: config.input,
+      outputSchema: config.outputSchema,
+      output: config.output
+    });
   }
-  return value;
-}
-function toPrettyJson(value) {
-  const str = stringify(value);
-  try {
-    const parsed = JSON.parse(str);
-    return JSON.stringify(parsed, null, 2);
-  } catch {
-    return str;
+  getName() {
+    return this._config.name;
   }
-}
-function formatDiffParts(parts) {
-  const lines = [];
-  for (const part of parts) {
-    const prefix = part.added ? "+ " : part.removed ? "- " : "";
-    const partLines = part.value.split("\n");
-    for (let i = 0; i < partLines.length; i++) {
-      const line = partLines[i];
-      if (i === partLines.length - 1 && line === "")
-        continue;
-      lines.push(prefix + line);
-    }
+  getDisplayName() {
+    return this._config.displayName;
   }
-  return lines.join("\n");
-}
-function createDiffString(expected, actual, diffOptions) {
-  const expectedProcessed = preprocessForDiff(expected, diffOptions);
-  const actualProcessed = preprocessForDiff(actual, diffOptions);
-  if (diffOptions?.keysOnly) {
-    const expectedKeys = JSON.stringify(extractKeys(expectedProcessed), null, 2);
-    const actualKeys = JSON.stringify(extractKeys(actualProcessed), null, 2);
-    const parts2 = diffLines(expectedKeys, actualKeys);
-    return formatDiffParts(parts2);
+  getDisplayLabel() {
+    return this._config.displayName ?? this._config.name;
   }
-  const expectedStr = toPrettyJson(expectedProcessed);
-  const actualStr = toPrettyJson(actualProcessed);
-  if (expectedStr === actualStr) {
-    return "";
+  getTags() {
+    return this._config.tags;
   }
-  const parts = diffLines(expectedStr, actualStr);
-  if (diffOptions?.outputNewOnly) {
-    const filtered = parts.filter((p) => p.added === true);
-    return formatDiffParts(filtered);
+  getInputSchema() {
+    return this._config.inputSchema;
   }
-  return formatDiffParts(parts);
-}
-function extractKeys(value) {
-  if (value === null || typeof value !== "object") {
-    return "\xB7";
+  getInput() {
+    return resolve(this._config.input);
   }
-  if (Array.isArray(value)) {
-    return value.map(extractKeys);
+  getOutputSchema() {
+    return this._config.outputSchema;
   }
-  const result = {};
-  for (const [k, v] of Object.entries(value)) {
-    result[k] = extractKeys(v);
+  getOutput() {
+    if (this._config.output === void 0) {
+      return void 0;
+    }
+    return resolve(this._config.output);
   }
-  return result;
+};
+function getTestCaseDisplayLabel(testCase) {
+  if (typeof testCase.getDisplayLabel === "function") {
+    return testCase.getDisplayLabel();
+  }
+  return typeof testCase.getName === "function" ? testCase.getName() : "";
 }
-function formatLogMessage(msg) {
-  if (typeof msg === "string")
-    return msg;
-  if (msg instanceof Error)
-    return msg.stack ?? msg.message;
+function getTestCaseTagList(testCase) {
+  return typeof testCase.getTags === "function" ? [...testCase.getTags()] : [];
+}
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = resolve$1(config.artifactDirectory);
+  let entries;
   try {
-    if (msg !== null && typeof msg === "object") {
-      return JSON.stringify(msg, null, 2);
-    }
-    return String(msg);
+    entries = await readdir(baseDir);
   } catch {
-    return String(msg);
+    return [];
   }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
 }
-function createLogEntry(message, options) {
-  return {
-    type: "log",
-    label: options?.label,
-    message: formatLogMessage(message)
-  };
-}
-function getLogLines(entry) {
-  return entry.message.split("\n");
-}
-function createDiffLogEntry(expected, actual, options) {
-  const { label, ...diffOpts } = options ?? {};
-  const diff = createDiffString(expected, actual, diffOpts);
-  return {
-    type: "diff",
-    label,
-    expected,
-    actual,
-    diff: diff || "(no differences)"
-  };
-}
-function printJsonDiff(expected, actual, options = {}) {
-  const { color = true, ...diffOpts } = options;
-  const diff = createDiffString(expected, actual, diffOpts);
-  if (color) {
-    const lines = diff.split("\n").map((line) => {
-      const trimmed = line.trimStart();
-      if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
-        return `\x1B[31m${line}\x1B[0m`;
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
       }
-      if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
-        return `\x1B[32m${line}\x1B[0m`;
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
       }
-      return line;
-    });
-    const colored = lines.join("\n");
-    console.log(colored || "(no differences)");
-    return colored;
+    } catch {
+    }
   }
-  console.log(diff || "(no differences)");
-  return diff;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
 }
 
 // src/runner/config.ts
@@ -862,6 +1166,7 @@ var defaultRunnerConfig = {
     rootDir: process.cwd(),
     datasetSuffixes: [".dataset.ts", ".dataset.tsx", ".dataset.js", ".dataset.mjs"],
     evaluatorSuffixes: [".evaluator.ts", ".evaluator.tsx", ".evaluator.js", ".evaluator.mjs"],
+    runConfigSuffixes: [".run-config.ts", ".run-config.tsx", ".run-config.js", ".run-config.mjs"],
     testCaseSuffixes: [".test-case.ts", ".test-case.tsx", ".test-case.js", ".test-case.mjs"],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
@@ -887,6 +1192,11 @@ function toRunnerConfigOverrides(config) {
   } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
     discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
   }
+  if (rawDiscovery?.runConfigFilePatterns !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigFilePatterns;
+  } else if (rawDiscovery?.runConfigSuffixes !== void 0) {
+    discovery.runConfigSuffixes = rawDiscovery.runConfigSuffixes;
+  }
   if (rawDiscovery?.testCaseFilePatterns !== void 0) {
     discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
   } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
@@ -985,6 +1295,9 @@ function isDatasetLike(value) {
 function isEvaluatorLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "resolveContext") && hasMethod(value, "getEvaluateFn");
 }
+function isRunConfigLike(value) {
+  return hasMethod(value, "getName") && hasMethod(value, "getRuns") && typeof value.getRuns === "function";
+}
 function isTestCaseLike(value) {
   return hasMethod(value, "getName") && hasMethod(value, "getTags") && hasMethod(value, "getInput");
 }
@@ -1073,6 +1386,23 @@ async function collectEvaluatorsFromFiles(config) {
   );
   return found.flat();
 }
+async function collectRunConfigsFromFiles(config) {
+  const files = await walkDirectory(config.rootDir, config.excludeDirectories);
+  const matched = files.filter((filePath) => hasOneSuffix(filePath, config.runConfigSuffixes));
+  const found = await Promise.all(
+    matched.map(async (absolutePath) => {
+      const exports = await loadModuleExports(absolutePath);
+      const runConfigs = exports.filter(isRunConfigLike);
+      const relPath = relative(config.rootDir, absolutePath);
+      return runConfigs.map((runConfig) => ({
+        id: runConfig.getName(),
+        filePath: relPath,
+        runConfig
+      }));
+    })
+  );
+  return found.flat();
+}
 async function collectTestCasesFromFiles(config) {
   const files = await walkDirectory(config.rootDir, config.excludeDirectories);
   const matched = files.filter((filePath) => hasOneSuffix(filePath, config.testCaseSuffixes));
@@ -1165,15 +1495,17 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
-function buildEvaluationUnits(testCases) {
+function buildEvaluationUnits(testCases, repetitionCount) {
+  const count = Math.max(1, repetitionCount);
   const units = [];
   for (const testCaseItem of testCases) {
-    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    for (let r = 0; r < rerunTotal; r++) {
+    const repetitionId = `rep-${randomUUID()}`;
+    for (let r = 0; r < count; r++) {
       units.push({
         testCaseItem,
-        rerunIndex: r + 1,
-        rerunTotal
+        repetitionId,
+        repetitionIndex: r + 1,
+        repetitionCount: count
       });
     }
   }
@@ -1186,7 +1518,7 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
   return join(artifactDirectory, `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`);
 }
 function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
-  const { testCaseItem, rerunIndex, rerunTotal } = unit;
+  const { testCaseItem, repetitionId, repetitionIndex, repetitionCount } = unit;
   return Effect.gen(function* () {
     const evaluatorRunId = `run-${randomUUID()}`;
     const started = Date.now();
@@ -1195,11 +1527,12 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       type: "TestCaseStarted",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       startedTestCases: startedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal
+      repetitionId,
+      repetitionIndex,
+      repetitionCount
     });
     const evaluatorScores = [];
     let testCaseError;
@@ -1233,8 +1566,15 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
               meta: {
                 triggerId: task.triggerId,
                 runId: evaluatorRunId,
-                datasetId: task.datasetId
+                datasetName: task.dataset.getDisplayLabel(),
+                repetitionId,
+                repetitionIndex,
+                repetitionCount,
+                runConfigName: task.runConfigName
               },
+              testCaseTags: getTestCaseTagList(testCaseItem.testCase),
+              runConfigTags: task.runConfigTags,
+              evaluatorTags: getEvaluatorTagList(evaluator),
               logDiff,
               log,
               createError
@@ -1277,18 +1617,19 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
         });
       }
     }
-    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+    const repetitionPassedThis = evaluatorScores.every((s) => s.passed);
     const completedEvaluations = yield* Ref.modify(completedRef, (n) => [n + 1, n + 1]);
     const progressEvent = {
       type: "TestCaseProgress",
       runId: task.runId,
       testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
+      testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
       completedTestCases: completedEvaluations,
       totalTestCases: totalEvaluations,
-      rerunIndex,
-      rerunTotal,
-      passed: rerunPassedThis,
+      repetitionId,
+      repetitionIndex,
+      repetitionCount,
+      passed: repetitionPassedThis,
       durationMs: Date.now() - started,
       evaluatorScores,
       output,
@@ -1309,9 +1650,9 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
       (map) => {
         const key = testCaseItem.id;
         const existing = map.get(key) ?? { completedCount: 0, results: [] };
-        const newResults = [...existing.results, rerunPassedThis];
+        const newResults = [...existing.results, repetitionPassedThis];
         const newCompletedCount = existing.completedCount + 1;
-        const isLast = newCompletedCount === rerunTotal;
+        const isLast = newCompletedCount === repetitionCount;
         const newMap = new Map(map);
         newMap.set(key, {
           completedCount: newCompletedCount,
@@ -1348,10 +1689,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  const totalEvaluations = task.testCases.reduce(
-    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-    0
-  );
+  const totalEvaluations = task.testCases.length * Math.max(1, task.repetitions);
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* Ref.make(0);
   const startedRef = yield* Ref.make(0);
@@ -1360,7 +1698,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   const testCaseResultsRef = yield* Ref.make(
     /* @__PURE__ */ new Map()
   );
-  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const evaluationUnits = buildEvaluationUnits(task.testCases, task.repetitions);
   const processEvaluation = (unit) => processOneEvaluation(
     task,
     unit,
@@ -1374,11 +1712,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     failedRef,
     testCaseResultsRef
   );
-  yield* Effect.forEach(
-    evaluationUnits,
-    processEvaluation,
-    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
-  );
+  const globalSem = task.globalEvaluationSemaphore;
+  if (globalSem !== void 0) {
+    yield* Effect.forEach(
+      evaluationUnits,
+      (unit) => globalSem.withPermits(1)(processEvaluation(unit)),
+      { concurrency: "unbounded", discard: true }
+    );
+  } else {
+    yield* Effect.forEach(
+      evaluationUnits,
+      processEvaluation,
+      maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+    );
+  }
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
     Ref.get(completedRef),
     Ref.get(passedRef),
@@ -1414,125 +1761,34 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     artifactPath: task.snapshot.artifactPath
   });
 });
-async function loadRunSnapshotsFromArtifacts(config) {
-  const baseDir = resolve$1(config.artifactDirectory);
-  let entries;
-  try {
-    entries = await readdir(baseDir);
-  } catch {
-    return [];
-  }
-  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
-  const snapshots = [];
-  for (const fileName of jsonlFiles) {
-    const filePath = join(baseDir, fileName);
-    try {
-      const snapshot = await parseArtifactToSnapshot(filePath, config);
-      if (snapshot) {
-        snapshots.push(snapshot);
-      }
-    } catch {
-    }
-  }
-  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
-}
-async function parseArtifactToSnapshot(filePath, _config) {
-  const content = await readFile(filePath, "utf8");
-  const lines = content.split("\n").filter((line) => line.trim().length > 0);
-  if (lines.length === 0) {
-    return null;
-  }
-  let runQueued = null;
-  let runCompleted = null;
-  let runFailed = null;
-  let runStarted = null;
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      const type = event.type;
-      if (type === "RunQueued") {
-        runQueued = {
-          runId: event.runId,
-          datasetId: event.datasetId,
-          datasetName: event.datasetName,
-          evaluatorIds: event.evaluatorIds,
-          totalTestCases: event.totalTestCases ?? 0,
-          artifactPath: event.artifactPath ?? filePath,
-          ts: event.ts
-        };
-      }
-      if (type === "RunStarted") {
-        runStarted = { startedAt: event.startedAt };
-      }
-      if (type === "RunCompleted") {
-        runCompleted = {
-          passedTestCases: event.passedTestCases,
-          failedTestCases: event.failedTestCases,
-          totalTestCases: event.totalTestCases,
-          finishedAt: event.finishedAt
-        };
-      }
-      if (type === "RunFailed") {
-        runFailed = {
-          finishedAt: event.finishedAt,
-          errorMessage: event.errorMessage
-        };
-      }
-    } catch {
-    }
+
+// src/runner/name-pattern.ts
+function parseRegexLiteral(pattern) {
+  if (!pattern.startsWith("/")) {
+    return void 0;
   }
-  if (!runQueued) {
-    return null;
+  const lastSlash = pattern.lastIndexOf("/");
+  if (lastSlash <= 0) {
+    return void 0;
   }
-  const artifactPath = filePath;
-  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
-  const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
-  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
-  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
-    runId: runQueued.runId,
-    datasetId: runQueued.datasetId,
-    datasetName: runQueued.datasetName,
-    evaluatorIds: runQueued.evaluatorIds,
-    queuedAt: runQueued.ts ?? 0,
-    startedAt: runStarted?.startedAt,
-    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
-    totalTestCases: runQueued.totalTestCases,
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
-    status,
-    artifactPath,
-    errorMessage: runFailed?.errorMessage
+    source: pattern.slice(1, lastSlash),
+    flags: pattern.slice(lastSlash + 1)
   };
 }
-function aggregateTestCaseProgress(lines) {
-  let completedTestCases = 0;
-  const testCasePassedBy = /* @__PURE__ */ new Map();
-  for (const line of lines) {
-    try {
-      const event = JSON.parse(line);
-      if (event.type === "TestCaseProgress") {
-        const ev = event;
-        completedTestCases = ev.completedTestCases ?? completedTestCases;
-        const id = ev.testCaseId;
-        const current = testCasePassedBy.get(id);
-        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
-      }
-    } catch {
-    }
+function createNameMatcher(pattern) {
+  const normalizedPattern = pattern.trim();
+  const regexLiteral = parseRegexLiteral(normalizedPattern);
+  if (regexLiteral) {
+    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
+    return (value) => regex.test(value);
   }
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const passed of testCasePassedBy.values()) {
-    if (passed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
+  if (normalizedPattern.includes("*")) {
+    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    const regex = new RegExp(`^${escaped}$`, "i");
+    return (value) => regex.test(value);
   }
-  return { completedTestCases, passedTestCases, failedTestCases };
+  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
 async function appendJsonLine(artifactPath, payload) {
   await mkdir(dirname(artifactPath), { recursive: true });
@@ -1591,32 +1847,12 @@ function searchCollectedTestCases(all, query) {
 }
 
 // src/runner/api.ts
-function parseRegexLiteral(pattern) {
-  if (!pattern.startsWith("/")) {
-    return void 0;
-  }
-  const lastSlash = pattern.lastIndexOf("/");
-  if (lastSlash <= 0) {
-    return void 0;
-  }
-  return {
-    source: pattern.slice(1, lastSlash),
-    flags: pattern.slice(lastSlash + 1)
-  };
-}
-function createNameMatcher(pattern) {
-  const normalizedPattern = pattern.trim();
-  const regexLiteral = parseRegexLiteral(normalizedPattern);
-  if (regexLiteral) {
-    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);
-    return (value) => regex.test(value);
-  }
-  if (normalizedPattern.includes("*")) {
-    const escaped = normalizedPattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-    const regex = new RegExp(`^${escaped}$`, "i");
-    return (value) => regex.test(value);
+function normalizeRunRepetitions(value) {
+  const n = value ?? 1;
+  if (!Number.isInteger(n) || n < 1) {
+    throw new Error(`repetitions must be a positive integer, got ${String(value)}`);
   }
-  return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
+  return n;
 }
 function mergeRunnerOverrides(base, next) {
   if (!base) {
@@ -1651,6 +1887,7 @@ var EffectRunner = class {
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
+    this.runConfigsById = /* @__PURE__ */ new Map();
     this.schedulerFiber = Effect.runFork(this.createSchedulerEffect());
     this.persistenceFiber = Effect.runFork(
       createPersistenceWorker(this.persistenceQueue)
@@ -1691,6 +1928,137 @@ var EffectRunner = class {
       (item) => matcher(item.evaluator.getName() ?? "")
     );
   }
+  async collectRunConfigs() {
+    const runConfigs = await collectRunConfigsFromFiles(this.config.discovery);
+    this.runConfigsById.clear();
+    const byNameLower = /* @__PURE__ */ new Map();
+    for (const item of runConfigs) {
+      const id = item.runConfig.getName();
+      const lower = id.toLowerCase();
+      const prev = byNameLower.get(lower);
+      if (prev !== void 0 && prev.filePath !== item.filePath) {
+        throw new Error(
+          `Duplicate RunConfig name "${id}" (matches "${prev.runConfig.getName()}" case-insensitively): ${prev.filePath} and ${item.filePath}`
+        );
+      }
+      byNameLower.set(lower, item);
+      this.runConfigsById.set(id, item);
+    }
+    return runConfigs;
+  }
+  async resolveRunConfigByName(name) {
+    if (this.runConfigsById.size === 0) {
+      await this.collectRunConfigs();
+    }
+    const key = validateRunConfigName(name, `RunConfig "${name.trim()}"`);
+    const keyLower = key.toLowerCase();
+    const matches = Array.from(this.runConfigsById.values()).filter(
+      (item) => item.runConfig.getName().toLowerCase() === keyLower
+    );
+    if (matches.length === 0) {
+      return void 0;
+    }
+    if (matches.length > 1) {
+      throw new Error(
+        `Multiple RunConfigs named "${name}": ${matches.map((m) => m.filePath).join(", ")}`
+      );
+    }
+    return matches[0];
+  }
+  async expandRunConfigToJobs(collected) {
+    if (this.datasetsById.size === 0) {
+      await this.collectDatasets();
+    }
+    if (this.evaluatorsById.size === 0) {
+      await this.collectEvaluators();
+    }
+    const rcName = collected.runConfig.getName();
+    const jobs = [];
+    const runs = collected.runConfig.getRuns();
+    for (const [i, row] of runs.entries()) {
+      const dsCollected = Array.from(this.datasetsById.values()).find(
+        (d) => d.dataset === row.dataset
+      );
+      if (!dsCollected) {
+        throw new Error(
+          `RunConfig "${rcName}" run[${i}]: dataset "${row.dataset.getDisplayLabel()}" was not found among discovered dataset exports (import the same module instances the scanner loads).`
+        );
+      }
+      let evaluatorIds;
+      if ("evaluatorPattern" in row && typeof row.evaluatorPattern === "string") {
+        const matcher = createNameMatcher(row.evaluatorPattern);
+        const matched = Array.from(this.evaluatorsById.values()).filter(
+          (item) => matcher(item.evaluator.getName() ?? "")
+        );
+        if (matched.length === 0) {
+          throw new Error(
+            `RunConfig "${rcName}" run[${i}]: no evaluator matched pattern "${row.evaluatorPattern}"`
+          );
+        }
+        evaluatorIds = matched.map((item) => item.id);
+      } else {
+        const evaluators = row.evaluators;
+        evaluatorIds = [];
+        for (const ev of evaluators) {
+          const found = Array.from(this.evaluatorsById.values()).find(
+            (item) => item.evaluator === ev
+          );
+          if (!found) {
+            throw new Error(
+              `RunConfig "${rcName}" run[${i}]: evaluator "${getEvaluatorDisplayLabel(ev) ?? "unknown"}" was not found among discovered evaluator exports`
+            );
+          }
+          evaluatorIds.push(found.id);
+        }
+      }
+      const repetitions = "repetitions" in row && row.repetitions !== void 0 ? row.repetitions : 1;
+      jobs.push({
+        datasetId: dsCollected.id,
+        evaluatorIds,
+        runConfigName: rcName,
+        runConfigDisplayLabel: collected.runConfig.getDisplayLabel(),
+        runConfigTags: collected.runConfig.getTags(),
+        repetitions
+      });
+    }
+    return jobs;
+  }
+  async expandRunConfigNamesToJobs(names) {
+    const jobs = [];
+    for (const name of names) {
+      const collected = await this.resolveRunConfigByName(name);
+      if (!collected) {
+        const known = await this.collectRunConfigs();
+        const available = known.map((r) => r.runConfig.getName()).sort();
+        throw new Error(
+          available.length > 0 ? `RunConfig "${name}" not found. Available RunConfigs: ${available.join(", ")}` : `RunConfig "${name}" not found and no RunConfigs were discovered.`
+        );
+      }
+      jobs.push(...await this.expandRunConfigToJobs(collected));
+    }
+    return jobs;
+  }
+  async runDatasetJobsWithSharedConcurrency(request) {
+    const globalConcurrency = Math.max(1, request.globalConcurrency);
+    const sem = Effect.unsafeMakeSemaphore(globalConcurrency);
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const snapshots = [];
+    for (const job of request.jobs) {
+      snapshots.push(
+        await this.startDatasetRun({
+          datasetId: job.datasetId,
+          evaluatorIds: job.evaluatorIds,
+          triggerId,
+          maxConcurrency: this.config.maxConcurrency ?? 1,
+          globalEvaluationSemaphore: sem,
+          runConfigName: job.runConfigName,
+          runConfigTags: job.runConfigTags,
+          repetitions: job.repetitions
+        })
+      );
+    }
+    return snapshots;
+  }
   async searchTestCases(query) {
     const testCases = await collectTestCasesFromFiles(this.config.discovery);
     return searchCollectedTestCases(testCases, query);
@@ -1709,36 +2077,46 @@ var EffectRunner = class {
     );
   }
   async runDatasetWith(request) {
+    const runConfigName = validateRunConfigName(
+      request.runConfigName,
+      "runDatasetWith.runConfigName"
+    );
+    return this.startDatasetRun({
+      datasetId: request.datasetId,
+      evaluatorIds: request.evaluatorIds,
+      triggerId: request.triggerId,
+      maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
+      repetitions: request.repetitions,
+      runConfigName,
+      runConfigTags: request.runConfigTags
+    });
+  }
+  async startDatasetRun(params) {
     if (this.datasetsById.size === 0) {
       await this.collectDatasets();
     }
     if (this.evaluatorsById.size === 0) {
       await this.collectEvaluators();
     }
-    const dataset = this.datasetsById.get(request.datasetId);
+    const dataset = this.datasetsById.get(params.datasetId);
     if (!dataset) {
-      throw new Error(`Unknown dataset: ${request.datasetId}`);
+      throw new Error(`Unknown dataset: ${params.datasetId}`);
     }
-    const selectedEvaluators = request.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
+    const selectedEvaluators = params.evaluatorIds.map((id) => this.evaluatorsById.get(id)).filter((value) => Boolean(value)).map((value) => ({ id: value.id, evaluator: value.evaluator }));
     if (selectedEvaluators.length === 0) {
       throw new Error("No evaluators selected for run");
     }
-    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
-    const totalEvaluations = selectedTestCases.reduce(
-      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
-      0
-    );
-    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
+    const selectedTestCases = await this.collectDatasetTestCases(params.datasetId);
+    const repetitions = normalizeRunRepetitions(params.repetitions);
+    const totalEvaluations = selectedTestCases.length * repetitions;
+    const runConfigTags = [...params.runConfigTags ?? []];
+    const triggerId = params.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
-    const artifactPath = createArtifactPath(
-      this.config.artifactDirectory,
-      request.datasetId,
-      runId
-    );
+    const artifactPath = createArtifactPath(this.config.artifactDirectory, params.datasetId, runId);
     const snapshot = {
       runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetId: params.datasetId,
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
       totalTestCases: totalEvaluations,
@@ -1758,8 +2136,8 @@ var EffectRunner = class {
     const queuedEvent = {
       type: "RunQueued",
       runId,
-      datasetId: request.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetId: params.datasetId,
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       totalTestCases: totalEvaluations,
       artifactPath
@@ -1772,17 +2150,20 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
-    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
         triggerId,
-        datasetId: request.datasetId,
+        datasetId: params.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
         snapshot,
-        maxConcurrency
+        maxConcurrency: params.maxConcurrency,
+        globalEvaluationSemaphore: params.globalEvaluationSemaphore,
+        runConfigName: params.runConfigName,
+        runConfigTags,
+        repetitions
       })
     );
     return snapshot;
@@ -1854,6 +2235,11 @@ var EffectRunner = class {
   }
 };
 
-export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, deltaScore, formatScoreData, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+// src/runner/events.ts
+var PROGRAMMATIC_RUN_CONFIG = {
+  runConfigName: "programmatic"
+};
+
+export { Dataset, DatasetNameSchema, Evaluator, EvaluatorNameSchema, Metric, PROGRAMMATIC_RUN_CONFIG, RunConfig, RunConfigNameSchema, Score, TagSet, TestCase, TestCaseNameSchema, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, deltaScore, formatScoreData, getDatasetDisplayLabel, getEvaluatorDisplayLabel, getEvaluatorTagList, getLogLines, getMetricById, getScoreById, getTestCaseDisplayLabel, getTestCaseTagList, latencyMetric, loadMockData, loadRunnerData, normalizeOptionalDisplayName, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, validateDatasetName, validateEvaluatorName, validateRunConfigName, validateTestCaseName, withRunnerConfig };
 //# sourceMappingURL=out.js.map
 //# sourceMappingURL=index.js.map