@m4trix/evals 0.26.0 → 0.28.0

package/README.md

@@ -62,7 +62,8 @@ export default defineConfig((): ConfigType => ({
 import { Dataset } from '@m4trix/evals';
 
 export const myDataset = Dataset.define({
-  name: 'My Dataset',
+  name: 'my-dataset',
+  displayName: 'My Dataset',
   includedTags: ['demo'],
 });
 ```
@@ -79,6 +80,7 @@ export const myEvaluator = Evaluator.define({
   inputSchema,
   outputSchema: S.Unknown,
   scoreSchema: S.Struct({ scores: S.Array(S.Unknown) }),
+  // optional: tags: ['suite-a'],
 }).evaluate(async ({ input, ctx: _ctx, output, createError }) => {
   const start = Date.now();
   const value = 85;
@@ -131,13 +133,15 @@ export const myTestCase = TestCase.describe({
 });
 ```
 
+`tags` is optional; omit it when the test case has no declared labels. Evaluators read them as `meta.testCaseTags`.
+
 ### 4) RunConfig (optional)
 
 Group several dataset/evaluator runs under one named config. Each row is either
 `evaluators: [...]` (same module instances discovery loads) or `evaluatorPattern: "..."`
 (wildcard / regex rules from `RunnerApi.resolveEvaluatorsByNamePattern`). Multiple jobs share one `--concurrency` cap.
 
-Optional **`repetitions`** on a row (default `1`) runs each matching test case that many times. Every execution in that group shares the same **`repetitionId`** in the evaluator callback **`meta`**, with **`repetitionIndex`** / **`repetitionCount`**. Evaluator **`meta`** includes **`runConfigName`**: the **`RunConfig`** name (or **`programmatic`** from **`PROGRAMMATIC_RUN_CONFIG`** for API/TUI-only **`runDatasetWith`**). Names may use **kebab-case**, **snake_case**, **camelCase**, etc. (letters, digits, `_`, `-` only, no spaces); resolution is **case-insensitive**.
+Optional **`repetitions`** on a row (default `1`) runs each matching test case that many times. Every execution in that group shares the same **`repetitionId`** in the evaluator callback **`meta`**, with **`repetitionIndex`** / **`repetitionCount`**. Evaluator **`meta`** includes **`datasetName`** (`Dataset.getDisplayLabel()` → `displayName ?? name`), **`runConfigName`** (the **`RunConfig`** id or **`programmatic`** from **`PROGRAMMATIC_RUN_CONFIG`** for API/TUI-only **`runDatasetWith`**), optional **`experimentName`**, and declared tag lists **`testCaseTags`**, **`runConfigTags`**, and **`evaluatorTags`** (empty arrays when unset). **`Dataset`** **`includedTags` / `excludedTags`** only filter which test cases belong to a dataset; they are not the same as **`TestCase.describe({ tags })`** or **`Evaluator.define({ tags })`**, which label the case/evaluator and show up in **`meta`**. **`Dataset`** and **`TestCase`** follow the same naming convention as **`RunConfig`**: **`name`** is the stable id; optional **`displayName`** is unrestricted for UI. Names may use **kebab-case**, **snake_case**, **camelCase**, etc. (letters, digits, `_`, `-` only, no spaces); resolution is **case-insensitive**.
 
 ```ts
 import { RunConfig } from '@m4trix/evals';
@@ -159,13 +163,13 @@ export const nightly = RunConfig.define({
 eval-agents-simple run --run-config "nightly"
 ```
 
-Repeat **`--run-config`** to queue several configs; jobs share one **`--concurrency`** cap.
+Repeat **`--run-config`** to queue several configs; jobs share one **`--concurrency`** cap. Use **`--experiment <name>`** to set **`meta.experimentName`** for every evaluator in that CLI run (any non-empty string; trimmed).
 
 ## CLI Commands
 
 - `eval-agents`: interactive CLI (starts runs with synthetic meta `programmatic` / `Programmatic`)
 - `eval-agents-simple run --run-config "<RunConfig name>"` (repeatable; case-insensitive match); add **`--ci`** to exit with code **1** if any test case fails
-- `eval-agents-simple generate --dataset "<dataset name>"`
+- `eval-agents-simple generate --dataset "<dataset id>"` (canonical **`Dataset` `name`**, case-insensitive)
 
 ## Default Discovery and Artifacts
 

package/dist/cli-simple.cjs

@@ -55,6 +55,7 @@ function makeEntityIdSchema(brand, label) {
 var RunConfigNameSchema = makeEntityIdSchema("RunConfigName", "RunConfig name");
 makeEntityIdSchema("EvaluatorName", "Evaluator name");
 makeEntityIdSchema("TestCaseName", "Test case name");
+makeEntityIdSchema("DatasetName", "Dataset name");
 function validateWithSchema(schema, raw, context) {
   const trimmed = raw.trim();
   const decode = effect.Schema.decodeUnknownEither(
@@ -611,6 +612,14 @@ function getTestCaseTagList(testCase) {
   return typeof testCase.getTags === "function" ? [...testCase.getTags()] : [];
 }
 
+// src/evals/dataset.ts
+function getDatasetDisplayLabel(dataset) {
+  if (typeof dataset.getDisplayLabel === "function") {
+    return dataset.getDisplayLabel();
+  }
+  return typeof dataset.getName === "function" ? dataset.getName() : "";
+}
+
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
 var Metric = {
@@ -1004,15 +1013,16 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
               meta: {
                 triggerId: task.triggerId,
                 runId: evaluatorRunId,
-                datasetId: task.datasetId,
+                datasetName: task.dataset.getDisplayLabel(),
                 repetitionId,
                 repetitionIndex,
                 repetitionCount,
-                runConfigName: task.runConfigName
+                runConfigName: task.runConfigName,
+                ...task.experimentName !== void 0 && task.experimentName !== "" ? { experimentName: task.experimentName } : {},
+                testCaseTags: getTestCaseTagList(testCaseItem.testCase),
+                runConfigTags: task.runConfigTags,
+                evaluatorTags: getEvaluatorTagList(evaluator)
               },
-              testCaseTags: getTestCaseTagList(testCaseItem.testCase),
-              runConfigTags: task.runConfigTags,
-              evaluatorTags: getEvaluatorTagList(evaluator),
               logDiff,
               log,
               createError
@@ -1419,7 +1429,7 @@ var EffectRunner = class {
       );
       if (!dsCollected) {
         throw new Error(
-          `RunConfig "${rcName}" run[${i}]: dataset "${row.dataset.getName()}" was not found among discovered dataset exports (import the same module instances the scanner loads).`
+          `RunConfig "${rcName}" run[${i}]: dataset "${row.dataset.getDisplayLabel()}" was not found among discovered dataset exports (import the same module instances the scanner loads).`
         );
       }
       let evaluatorIds;
@@ -1491,7 +1501,8 @@ var EffectRunner = class {
           globalEvaluationSemaphore: sem,
           runConfigName: job.runConfigName,
           runConfigTags: job.runConfigTags,
-          repetitions: job.repetitions
+          repetitions: job.repetitions,
+          experimentName: request.experimentName
         })
       );
     }
@@ -1526,7 +1537,8 @@ var EffectRunner = class {
       maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
       repetitions: request.repetitions,
       runConfigName,
-      runConfigTags: request.runConfigTags
+      runConfigTags: request.runConfigTags,
+      experimentName: request.experimentName
     });
   }
   async startDatasetRun(params) {
@@ -1554,7 +1566,7 @@ var EffectRunner = class {
     const snapshot = {
       runId,
       datasetId: params.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
       totalTestCases: totalEvaluations,
@@ -1575,7 +1587,7 @@ var EffectRunner = class {
       type: "RunQueued",
       runId,
       datasetId: params.datasetId,
-      datasetName: dataset.dataset.getName(),
+      datasetName: dataset.dataset.getDisplayLabel(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       totalTestCases: totalEvaluations,
       artifactPath
@@ -1601,7 +1613,8 @@ var EffectRunner = class {
         globalEvaluationSemaphore: params.globalEvaluationSemaphore,
         runConfigName: params.runConfigName,
         runConfigTags,
-        repetitions
+        repetitions,
+        experimentName: params.experimentName
       })
     );
     return snapshot;
@@ -1721,6 +1734,17 @@ function parseSimpleCliArgs(argv) {
       index += 1;
       continue;
     }
+    if (token === "--experiment" && argv[index + 1]) {
+      const raw = argv[index + 1];
+      if (typeof raw === "string") {
+        const trimmed = raw.trim();
+        if (trimmed.length > 0) {
+          args.experimentName = trimmed;
+        }
+      }
+      index += 1;
+      continue;
+    }
     args.unknownArgs.push(token);
   }
   return args;
@@ -1728,12 +1752,13 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --run-config <name> [--run-config <name> ...] [--concurrency N] [--ci]",
-    "  eval-agents-simple generate --dataset <datasetName>",
+    "  eval-agents-simple run --run-config <name> [--run-config <name> ...] [--concurrency N] [--experiment <name>] [--ci]",
+    "  eval-agents-simple generate --dataset <datasetId>",
     "",
     "Options:",
     "  --ci                  With run: exit with code 1 if any test case fails.",
-    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential."
+    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential.",
+    "  --experiment <name>   With run: set evaluator meta.experimentName for this invocation."
   ].join("\n");
 }
 
@@ -1797,7 +1822,7 @@ function GenerateView({
       if (!cancelled) {
         setResult({
           count: payload.length,
-          datasetName: dataset.dataset.getName(),
+          datasetName: getDatasetDisplayLabel(dataset.dataset),
           outputPath
         });
         setTimeout(() => onComplete(), 200);
@@ -1858,7 +1883,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   const outputPath = createOutputPath(absoluteDatasetPath);
   await promises.writeFile(outputPath, `${JSON.stringify(payload, null, 2)}
 `, "utf8");
-  console.log(`Generated ${payload.length} test cases for dataset "${dataset.dataset.getName()}".`);
+  console.log(`Generated ${payload.length} test cases for dataset "${getDatasetDisplayLabel(dataset.dataset)}".`);
   console.log(`Wrote ${outputPath}`);
 }
 async function generateDatasetJsonCommandInk(runner, datasetName) {
@@ -2010,6 +2035,7 @@ function RunView({
   runner,
   runConfigNames,
   concurrency,
+  experimentName,
   onComplete
 }) {
   const [phase, setPhase] = React.useState("loading");
@@ -2179,7 +2205,8 @@ function RunView({
     });
     const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
       jobs,
-      globalConcurrency: concurrency
+      globalConcurrency: concurrency,
+      experimentName
     });
     for (let i = 0; i < snapshots.length; i += 1) {
       const snap = snapshots[i];
@@ -2236,7 +2263,7 @@ function RunView({
     setPhase("completed");
     const exitCode = failedTestCases > 0 ? 1 : 0;
     setTimeout(() => onComplete(void 0, exitCode), 200);
-  }, [runner, runConfigNames, concurrency, onComplete]);
+  }, [runner, runConfigNames, concurrency, experimentName, onComplete]);
   React.useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -2724,7 +2751,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency) {
+async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency, experimentName) {
   const jobs = await runner.expandRunConfigNamesToJobs(runConfigNames);
   if (jobs.length === 0) {
     throw new Error("No jobs expanded from RunConfigs.");
@@ -2933,7 +2960,8 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency)
   console.log("");
   const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
     jobs,
-    globalConcurrency: concurrency
+    globalConcurrency: concurrency,
+    experimentName
   });
   for (let i = 0; i < snapshots.length; i += 1) {
     const snap = snapshots[i];
@@ -3033,13 +3061,14 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency)
   }
   return failedTestCasesTotal > 0 ? 1 : 0;
 }
-async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency) {
+async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency, experimentName) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
       React__namespace.createElement(RunView, {
         runner,
         runConfigNames,
         concurrency,
+        experimentName,
         onComplete: (err, exitCode) => {
           app.unmount();
           if (err) {
@@ -3100,7 +3129,8 @@ async function main() {
       const exitCode = await (useInk ? runSimpleEvalRunConfigsInk : runSimpleEvalRunConfigsPlain)(
         runner,
         args.runConfigNames,
-        concurrency
+        concurrency,
+        args.experimentName
       );
       if (args.ci && exitCode !== 0) {
         process.exit(1);
@@ -3109,7 +3139,7 @@ async function main() {
     }
     const genDataset = args.datasetName;
     if (!genDataset) {
-      console.error("Missing required --dataset <datasetName> argument.");
+      console.error("Missing required --dataset <datasetId> argument.");
       printUsageAndExit(1);
     }
     await (useInk ? generateDatasetJsonCommandInk : generateDatasetJsonCommandPlain)(