@m4trix/evals 0.27.0 → 0.28.0

package/dist/cli-simple.js

@@ -991,11 +991,12 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
                 repetitionId,
                 repetitionIndex,
                 repetitionCount,
-                runConfigName: task.runConfigName
+                runConfigName: task.runConfigName,
+                ...task.experimentName !== void 0 && task.experimentName !== "" ? { experimentName: task.experimentName } : {},
+                testCaseTags: getTestCaseTagList(testCaseItem.testCase),
+                runConfigTags: task.runConfigTags,
+                evaluatorTags: getEvaluatorTagList(evaluator)
               },
-              testCaseTags: getTestCaseTagList(testCaseItem.testCase),
-              runConfigTags: task.runConfigTags,
-              evaluatorTags: getEvaluatorTagList(evaluator),
               logDiff,
               log,
               createError
@@ -1474,7 +1475,8 @@ var EffectRunner = class {
           globalEvaluationSemaphore: sem,
           runConfigName: job.runConfigName,
           runConfigTags: job.runConfigTags,
-          repetitions: job.repetitions
+          repetitions: job.repetitions,
+          experimentName: request.experimentName
         })
       );
     }
@@ -1509,7 +1511,8 @@ var EffectRunner = class {
       maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
       repetitions: request.repetitions,
       runConfigName,
-      runConfigTags: request.runConfigTags
+      runConfigTags: request.runConfigTags,
+      experimentName: request.experimentName
     });
   }
   async startDatasetRun(params) {
@@ -1584,7 +1587,8 @@ var EffectRunner = class {
         globalEvaluationSemaphore: params.globalEvaluationSemaphore,
         runConfigName: params.runConfigName,
         runConfigTags,
-        repetitions
+        repetitions,
+        experimentName: params.experimentName
       })
     );
     return snapshot;
@@ -1704,6 +1708,17 @@ function parseSimpleCliArgs(argv) {
       index += 1;
       continue;
     }
+    if (token === "--experiment" && argv[index + 1]) {
+      const raw = argv[index + 1];
+      if (typeof raw === "string") {
+        const trimmed = raw.trim();
+        if (trimmed.length > 0) {
+          args.experimentName = trimmed;
+        }
+      }
+      index += 1;
+      continue;
+    }
     args.unknownArgs.push(token);
   }
   return args;
@@ -1711,12 +1726,13 @@ function parseSimpleCliArgs(argv) {
 function getSimpleCliUsage() {
   return [
     "Usage:",
-    "  eval-agents-simple run --run-config <name> [--run-config <name> ...] [--concurrency N] [--ci]",
+    "  eval-agents-simple run --run-config <name> [--run-config <name> ...] [--concurrency N] [--experiment <name>] [--ci]",
     "  eval-agents-simple generate --dataset <datasetId>",
     "",
     "Options:",
     "  --ci                  With run: exit with code 1 if any test case fails.",
-    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential."
+    "  --concurrency, -c N   Max concurrent evaluations (default: 4). Use 1 for sequential.",
+    "  --experiment <name>   With run: set evaluator meta.experimentName for this invocation."
   ].join("\n");
 }
 
@@ -1993,6 +2009,7 @@ function RunView({
   runner,
   runConfigNames,
   concurrency,
+  experimentName,
   onComplete
 }) {
   const [phase, setPhase] = useState("loading");
@@ -2162,7 +2179,8 @@ function RunView({
     });
     const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
       jobs,
-      globalConcurrency: concurrency
+      globalConcurrency: concurrency,
+      experimentName
     });
     for (let i = 0; i < snapshots.length; i += 1) {
       const snap = snapshots[i];
@@ -2219,7 +2237,7 @@ function RunView({
     setPhase("completed");
     const exitCode = failedTestCases > 0 ? 1 : 0;
     setTimeout(() => onComplete(void 0, exitCode), 200);
-  }, [runner, runConfigNames, concurrency, onComplete]);
+  }, [runner, runConfigNames, concurrency, experimentName, onComplete]);
   useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -2707,7 +2725,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency) {
+async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency, experimentName) {
   const jobs = await runner.expandRunConfigNamesToJobs(runConfigNames);
   if (jobs.length === 0) {
     throw new Error("No jobs expanded from RunConfigs.");
@@ -2916,7 +2934,8 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency)
   console.log("");
   const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
     jobs,
-    globalConcurrency: concurrency
+    globalConcurrency: concurrency,
+    experimentName
   });
   for (let i = 0; i < snapshots.length; i += 1) {
     const snap = snapshots[i];
@@ -3016,13 +3035,14 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency)
   }
   return failedTestCasesTotal > 0 ? 1 : 0;
 }
-async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency) {
+async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency, experimentName) {
   return new Promise((resolve5, reject) => {
     const app = render(
       React.createElement(RunView, {
         runner,
         runConfigNames,
         concurrency,
+        experimentName,
         onComplete: (err, exitCode) => {
           app.unmount();
           if (err) {
@@ -3083,7 +3103,8 @@ async function main() {
       const exitCode = await (useInk ? runSimpleEvalRunConfigsInk : runSimpleEvalRunConfigsPlain)(
         runner,
         args.runConfigNames,
-        concurrency
+        concurrency,
+        args.experimentName
       );
       if (args.ci && exitCode !== 0) {
         process.exit(1);