@m4trix/evals 0.16.0 → 0.17.0

package/dist/index.d.ts

@@ -234,10 +234,23 @@ interface EvalMiddleware<TCtx> {
     name: string;
     resolve: () => TCtx | Promise<TCtx>;
 }
+interface EvaluateMeta {
+    /** Identifier of the trigger that started the run (for example, a CLI invocation). */
+    triggerId: string;
+    /**
+     * Identifier of the current test-case execution shared across all evaluators
+     * for this specific test-case run.
+     */
+    runId: string;
+    /** Identifier of the dataset currently being evaluated. */
+    datasetId: string;
+}
 interface EvaluateArgs<TInput, TOutput = unknown, TCtx = Record<string, never>> {
     input: TInput;
     ctx: TCtx;
     output?: TOutput;
+    /** Metadata about the current evaluator invocation. */
+    meta: EvaluateMeta;
     /** Records a diff for this test case; stored in run artifact and shown by CLI */
     logDiff: (expected: unknown, actual: unknown, options?: CreateDiffLogEntryOptions) => void;
     /** Logs a message or object for this test case; stored in run artifact and shown by CLI */
@@ -349,6 +362,11 @@ interface SearchTestCasesQuery {
     excludedPaths?: ReadonlyArray<string | RegExp>;
 }
 interface RunDatasetRequest {
+    /**
+     * Identifier for what triggered the run request (for example, a CLI command).
+     * When omitted, the runner generates one in the format `trg-[uuid]`.
+     */
+    triggerId?: string;
     datasetId: string;
     evaluatorIds: ReadonlyArray<string>;
     concurrency?: number;
@@ -469,4 +487,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, CreateDiffLogEntryOptions, Dataset, DiffLogEntry, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorLogEntry, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, JsonDiffOptions, LatencyData, LogEntry, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, CreateDiffLogEntryOptions, Dataset, DiffLogEntry, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, EvaluateMeta, Evaluator, EvaluatorLogEntry, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, JsonDiffOptions, LatencyData, LogEntry, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };

package/dist/index.js

@@ -1035,6 +1035,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
+      const evaluatorRunId = `run-${randomUUID()}`;
       const started = Date.now();
       const evaluatorScores = [];
       let testCaseError;
@@ -1061,6 +1062,11 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
+                meta: {
+                  triggerId: task.triggerId,
+                  runId: evaluatorRunId,
+                  datasetId: task.datasetId
+                },
                 logDiff,
                 log
               })
@@ -1520,6 +1526,7 @@ var EffectRunner = class {
       (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
       0
     );
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1561,6 +1568,7 @@ var EffectRunner = class {
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
+        triggerId,
         datasetId: request.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,