@m4trix/evals 0.28.0 → 0.30.0

package/README.md

@@ -141,7 +141,7 @@ Group several dataset/evaluator runs under one named config. Each row is either
 `evaluators: [...]` (same module instances discovery loads) or `evaluatorPattern: "..."`
 (wildcard / regex rules from `RunnerApi.resolveEvaluatorsByNamePattern`). Multiple jobs share one `--concurrency` cap.
 
-Optional **`repetitions`** on a row (default `1`) runs each matching test case that many times. Every execution in that group shares the same **`repetitionId`** in the evaluator callback **`meta`**, with **`repetitionIndex`** / **`repetitionCount`**. Evaluator **`meta`** includes **`datasetName`** (`Dataset.getDisplayLabel()` → `displayName ?? name`), **`runConfigName`** (the **`RunConfig`** id or **`programmatic`** from **`PROGRAMMATIC_RUN_CONFIG`** for API/TUI-only **`runDatasetWith`**), optional **`experimentName`**, and declared tag lists **`testCaseTags`**, **`runConfigTags`**, and **`evaluatorTags`** (empty arrays when unset). **`Dataset`** **`includedTags` / `excludedTags`** only filter which test cases belong to a dataset; they are not the same as **`TestCase.describe({ tags })`** or **`Evaluator.define({ tags })`**, which label the case/evaluator and show up in **`meta`**. **`Dataset`** and **`TestCase`** follow the same naming convention as **`RunConfig`**: **`name`** is the stable id; optional **`displayName`** is unrestricted for UI. Names may use **kebab-case**, **snake_case**, **camelCase**, etc. (letters, digits, `_`, `-` only, no spaces); resolution is **case-insensitive**.
+Optional **`repetitions`** on a row (default `1`) runs each matching test case that many times. Every execution in that group shares the same **`repetitionId`** in the evaluator callback **`meta`**, with **`repetitionIndex`** / **`repetitionCount`**. Evaluator **`meta`** includes **`triggerId`**, **`triggerTimestamp`** (ms since epoch when the run was triggered; the simple CLI sets this once at process start), **`datasetName`** (`Dataset.getDisplayLabel()` → `displayName ?? name`), **`testCaseId`** (discovery id, same as runner events), **`testCaseName`** (`TestCase.getDisplayLabel()` → `displayName ?? name`), **`runConfigName`** (the **`RunConfig`** id or **`programmatic`** from **`PROGRAMMATIC_RUN_CONFIG`** for API/TUI-only **`runDatasetWith`**), optional **`experimentName`**, and declared tag lists **`testCaseTags`**, **`runConfigTags`**, and **`evaluatorTags`** (empty arrays when unset). **`Dataset`** **`includedTags` / `excludedTags`** only filter which test cases belong to a dataset; they are not the same as **`TestCase.describe({ tags })`** or **`Evaluator.define({ tags })`**, which label the case/evaluator and show up in **`meta`**. **`Dataset`** and **`TestCase`** follow the same naming convention as **`RunConfig`**: **`name`** is the stable id; optional **`displayName`** is unrestricted for UI. Names may use **kebab-case**, **snake_case**, **camelCase**, etc. (letters, digits, `_`, `-` only, no spaces); resolution is **case-insensitive**.
 
 ```ts
 import { RunConfig } from '@m4trix/evals';

package/dist/cli-simple.cjs

@@ -1012,8 +1012,11 @@ function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persis
               output,
               meta: {
                 triggerId: task.triggerId,
+                triggerTimestamp: task.triggerTimestamp,
                 runId: evaluatorRunId,
                 datasetName: task.dataset.getDisplayLabel(),
+                testCaseId: testCaseItem.id,
+                testCaseName: getTestCaseDisplayLabel(testCaseItem.testCase),
                 repetitionId,
                 repetitionIndex,
                 repetitionCount,
@@ -1490,6 +1493,7 @@ var EffectRunner = class {
     const globalConcurrency = Math.max(1, request.globalConcurrency);
     const sem = effect.Effect.unsafeMakeSemaphore(globalConcurrency);
     const triggerId = request.triggerId ?? `trg-${crypto.randomUUID()}`;
+    const triggerTimestamp = request.triggerTimestamp ?? Date.now();
     const snapshots = [];
     for (const job of request.jobs) {
       snapshots.push(
@@ -1497,6 +1501,7 @@ var EffectRunner = class {
           datasetId: job.datasetId,
           evaluatorIds: job.evaluatorIds,
           triggerId,
+          triggerTimestamp,
           maxConcurrency: this.config.maxConcurrency ?? 1,
           globalEvaluationSemaphore: sem,
           runConfigName: job.runConfigName,
@@ -1534,6 +1539,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       evaluatorIds: request.evaluatorIds,
       triggerId: request.triggerId,
+      triggerTimestamp: request.triggerTimestamp ?? Date.now(),
       maxConcurrency: request.concurrency ?? this.config.maxConcurrency ?? 1,
       repetitions: request.repetitions,
       runConfigName,
@@ -1561,6 +1567,7 @@ var EffectRunner = class {
     const totalEvaluations = selectedTestCases.length * repetitions;
     const runConfigTags = [...params.runConfigTags ?? []];
     const triggerId = params.triggerId ?? `trg-${crypto.randomUUID()}`;
+    const triggerTimestamp = params.triggerTimestamp ?? Date.now();
     const runId = `run-${crypto.randomUUID()}`;
     const artifactPath = createArtifactPath(this.config.artifactDirectory, params.datasetId, runId);
     const snapshot = {
@@ -1604,6 +1611,7 @@ var EffectRunner = class {
       effect.Queue.offer(this.runQueue, {
         runId,
         triggerId,
+        triggerTimestamp,
         datasetId: params.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
@@ -2036,6 +2044,7 @@ function RunView({
   runConfigNames,
   concurrency,
   experimentName,
+  triggerTimestamp,
   onComplete
 }) {
   const [phase, setPhase] = React.useState("loading");
@@ -2206,7 +2215,8 @@ function RunView({
     const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
       jobs,
       globalConcurrency: concurrency,
-      experimentName
+      experimentName,
+      triggerTimestamp
     });
     for (let i = 0; i < snapshots.length; i += 1) {
       const snap = snapshots[i];
@@ -2263,7 +2273,7 @@ function RunView({
     setPhase("completed");
     const exitCode = failedTestCases > 0 ? 1 : 0;
     setTimeout(() => onComplete(void 0, exitCode), 200);
-  }, [runner, runConfigNames, concurrency, experimentName, onComplete]);
+  }, [runner, runConfigNames, concurrency, experimentName, triggerTimestamp, onComplete]);
   React.useEffect(() => {
     void runEval();
   }, [runEval]);
@@ -2751,7 +2761,7 @@ function formatEvaluatorScoreLine(name, scores, passed, metrics, options) {
   }
   return lines;
 }
-async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency, experimentName) {
+async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency, experimentName, triggerTimestamp) {
   const jobs = await runner.expandRunConfigNamesToJobs(runConfigNames);
   if (jobs.length === 0) {
     throw new Error("No jobs expanded from RunConfigs.");
@@ -2961,7 +2971,8 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency,
   const snapshots = await runner.runDatasetJobsWithSharedConcurrency({
     jobs,
     globalConcurrency: concurrency,
-    experimentName
+    experimentName,
+    triggerTimestamp
   });
   for (let i = 0; i < snapshots.length; i += 1) {
     const snap = snapshots[i];
@@ -3061,7 +3072,7 @@ async function runSimpleEvalRunConfigsPlain(runner, runConfigNames, concurrency,
   }
   return failedTestCasesTotal > 0 ? 1 : 0;
 }
-async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency, experimentName) {
+async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency, experimentName, triggerTimestamp) {
   return new Promise((resolve5, reject) => {
     const app = ink.render(
       React__namespace.createElement(RunView, {
@@ -3069,6 +3080,7 @@ async function runSimpleEvalRunConfigsInk(runner, runConfigNames, concurrency, e
         runConfigNames,
         concurrency,
         experimentName,
+        triggerTimestamp,
         onComplete: (err, exitCode) => {
           app.unmount();
           if (err) {
@@ -3126,11 +3138,13 @@ async function main() {
   try {
     if (args.command === "run") {
       const concurrency = args.concurrency ?? getDefaultConcurrency();
+      const triggerTimestamp = Date.now();
       const exitCode = await (useInk ? runSimpleEvalRunConfigsInk : runSimpleEvalRunConfigsPlain)(
         runner,
         args.runConfigNames,
         concurrency,
-        args.experimentName
+        args.experimentName,
+        triggerTimestamp
       );
       if (args.ci && exitCode !== 0) {
         process.exit(1);