@m4trix/evals 0.16.0 → 0.18.0

package/dist/index.d.ts

@@ -234,10 +234,23 @@ interface EvalMiddleware<TCtx> {
     name: string;
     resolve: () => TCtx | Promise<TCtx>;
 }
+interface EvaluateMeta {
+    /** Identifier of the trigger that started the run (for example, a CLI invocation). */
+    triggerId: string;
+    /**
+     * Identifier of the current test-case execution shared across all evaluators
+     * for this specific test-case run.
+     */
+    runId: string;
+    /** Identifier of the dataset currently being evaluated. */
+    datasetId: string;
+}
 interface EvaluateArgs<TInput, TOutput = unknown, TCtx = Record<string, never>> {
     input: TInput;
     ctx: TCtx;
     output?: TOutput;
+    /** Metadata about the current evaluator invocation. */
+    meta: EvaluateMeta;
     /** Records a diff for this test case; stored in run artifact and shown by CLI */
     logDiff: (expected: unknown, actual: unknown, options?: CreateDiffLogEntryOptions) => void;
     /** Logs a message or object for this test case; stored in run artifact and shown by CLI */
@@ -302,6 +315,8 @@ interface ScoreItem<TData = unknown> {
     readonly id: string;
     readonly data: TData;
     readonly passed?: boolean;
+    /** Attached def for formatting/aggregation without registry lookup (avoids n/a across module boundaries) */
+    readonly def?: ScoreDef<TData>;
 }
 interface FormatScoreOptions {
     isAggregated?: boolean;
@@ -310,19 +325,43 @@ interface ScoreDef<TData = unknown> {
     readonly id: string;
     readonly name?: string;
     readonly displayStrategy: ScoreDisplayStrategy;
-    readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
-    format(data: TData, options?: FormatScoreOptions): string;
+    readonly formatValue: (data: TData) => string;
+    readonly formatAggregate: (data: TData) => string;
+    readonly aggregateValues: (values: ReadonlyArray<TData>) => TData;
     make(data: TData, options?: {
         definePassed?: (data: TData) => boolean;
     }): ScoreItem<TData>;
 }
+/** Helper to format using the right method based on isAggregated (for consumers that need a single entry point) */
+declare function formatScoreData<TData>(def: ScoreDef<TData>, data: TData, options?: FormatScoreOptions): string;
 declare const Score: {
+    aggregate: {
+        /** Average numeric fields. Use for scores like { value, delta }. */
+        averageFields<K extends string>(fields: readonly K[]): (values: readonly Record<K, number>[]) => Record<K, number>;
+        /** Average `value` with sample std dev. Use for percent-style scores. */
+        averageWithVariance<T extends {
+            value: number;
+        }>(values: readonly T[]): T & {
+            stdDev?: number | undefined;
+            count: number;
+        };
+        /** All runs must pass. Use for binary scores. */
+        all<T_1 extends {
+            passed: boolean;
+        }>(values: readonly T_1[]): T_1 & {
+            passedCount?: number | undefined;
+            totalCount?: number | undefined;
+        };
+        /** Take last value (no aggregation). Use when aggregation is not meaningful. */
+        last<T_2>(values: readonly T_2[]): T_2;
+    };
     of<TData>(config: {
         id: string;
         name?: string | undefined;
         displayStrategy: ScoreDisplayStrategy;
-        format: (data: TData, options?: FormatScoreOptions) => string;
-        aggregate?: ((values: readonly TData[]) => TData) | undefined;
+        formatValue: (data: TData) => string;
+        formatAggregate: (data: TData) => string;
+        aggregateValues: (values: readonly TData[]) => TData;
     }): ScoreDef<TData>;
 };
 declare function getScoreById(id: string): ScoreDef<unknown> | undefined;
@@ -349,6 +388,11 @@ interface SearchTestCasesQuery {
     excludedPaths?: ReadonlyArray<string | RegExp>;
 }
 interface RunDatasetRequest {
+    /**
+     * Identifier for what triggered the run request (for example, a CLI command).
+     * When omitted, the runner generates one in the format `trg-[uuid]`.
+     */
+    triggerId?: string;
     datasetId: string;
     evaluatorIds: ReadonlyArray<string>;
     concurrency?: number;
@@ -462,6 +506,11 @@ interface PercentScoreData {
     count?: number;
 }
 declare const percentScore: ScoreDef<PercentScoreData>;
+interface DeltaScoreData {
+    value: number;
+    delta: number;
+}
+declare const deltaScore: ScoreDef<DeltaScoreData>;
 interface BinaryScoreData {
     passed: boolean;
     passedCount?: number;
@@ -469,4 +518,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, CreateDiffLogEntryOptions, Dataset, DiffLogEntry, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorLogEntry, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, JsonDiffOptions, LatencyData, LogEntry, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, CreateDiffLogEntryOptions, Dataset, DeltaScoreData, DiffLogEntry, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, EvaluateMeta, Evaluator, EvaluatorLogEntry, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, JsonDiffOptions, LatencyData, LogEntry, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, deltaScore, formatScoreData, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };

package/dist/index.js

@@ -513,20 +513,70 @@ function getMetricById(id) {
 
 // src/evals/score.ts
 var registry2 = /* @__PURE__ */ new Map();
+function formatScoreData(def, data, options) {
+  return options?.isAggregated ? def.formatAggregate(data) : def.formatValue(data);
+}
+var ScoreAggregate = {
+  /** Average numeric fields. Use for scores like { value, delta }. */
+  averageFields(fields) {
+    return (values) => {
+      const count = values.length || 1;
+      const result = {};
+      for (const field of fields) {
+        result[field] = values.reduce((s, v) => s + (v[field] ?? 0), 0) / count;
+      }
+      return result;
+    };
+  },
+  /** Average `value` with sample std dev. Use for percent-style scores. */
+  averageWithVariance(values) {
+    if (values.length === 0) {
+      return { value: 0, stdDev: void 0, count: 0 };
+    }
+    const sum = values.reduce((s, v) => s + v.value, 0);
+    const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+    const mean = sum / values.length;
+    let stdDev;
+    if (values.length >= 2) {
+      const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+      stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+    }
+    return { ...values[0], value: mean, stdDev, count: values.length };
+  },
+  /** All runs must pass. Use for binary scores. */
+  all(values) {
+    const total = values.length;
+    const passedCount = values.filter((v) => v.passed).length;
+    return {
+      ...values[0],
+      passed: total > 0 && values.every((v) => v.passed),
+      passedCount,
+      totalCount: total
+    };
+  },
+  /** Take last value (no aggregation). Use when aggregation is not meaningful. */
+  last(values) {
+    return values[values.length - 1] ?? {};
+  }
+};
 var Score = {
+  aggregate: ScoreAggregate,
   of(config) {
     const def = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
-      aggregate: config.aggregate,
-      format: config.format,
+      formatValue: config.formatValue,
+      formatAggregate: config.formatAggregate,
+      aggregateValues: config.aggregateValues,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
         return {
           id: config.id,
           data,
-          ...passed !== void 0 && { passed }
+          ...passed !== void 0 && { passed },
+          def
+          // Attach def so rendering/aggregation works without registry lookup
         };
       }
     };
@@ -539,29 +589,6 @@ function getScoreById(id) {
 }
 
 // src/evals/aggregators.ts
-function aggregateAverageWithVariance(values) {
-  if (values.length === 0) {
-    return { value: 0, count: 0 };
-  }
-  const sum = values.reduce((s, v) => s + v.value, 0);
-  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
-  const mean = sum / values.length;
-  let stdDev;
-  if (values.length >= 2) {
-    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
-    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
-  }
-  return { value: mean, stdDev, count: values.length };
-}
-function aggregateAll(values) {
-  const total = values.length;
-  const passedCount = values.filter((v) => v.passed).length;
-  return {
-    passed: total > 0 && values.every((v) => v.passed),
-    passedCount,
-    totalCount: total
-  };
-}
 function aggregateTokenCountSum(values) {
   const initial = {
     input: 0,
@@ -614,29 +641,31 @@ var percentScore = Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data, options) => {
-    if (options?.isAggregated) {
-      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
-    }
-    return data.value.toFixed(2);
-  },
-  aggregate: aggregateAverageWithVariance
+  formatValue: (data) => data.value.toFixed(2),
+  formatAggregate: (data) => data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`,
+  aggregateValues: Score.aggregate.averageWithVariance
+});
+var deltaScore = Score.of({
+  id: "delta",
+  name: "Delta",
+  displayStrategy: "number",
+  formatValue: (data) => `${data.value.toFixed(2)} (${data.delta >= 0 ? "+" : ""}${data.delta.toFixed(2)} vs baseline)`,
+  formatAggregate: (data) => `Avg: ${data.value.toFixed(2)} (Delta: ${data.delta >= 0 ? "+" : ""}${data.delta.toFixed(2)})`,
+  aggregateValues: Score.aggregate.averageFields(["value", "delta"])
 });
 var binaryScore = Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data, options) => {
-    if (options?.isAggregated) {
-      const base = data.passed ? "All: PASSED" : "Some: FAILED";
-      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
-        return `${base} (${data.passedCount}/${data.totalCount})`;
-      }
-      return base;
+  formatValue: (data) => data.passed ? "PASSED" : "NOT PASSED",
+  formatAggregate: (data) => {
+    const base = data.passed ? "All: PASSED" : "Some: FAILED";
+    if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+      return `${base} (${data.passedCount}/${data.totalCount})`;
     }
-    return data.passed ? "PASSED" : "NOT PASSED";
+    return base;
   },
-  aggregate: aggregateAll
+  aggregateValues: Score.aggregate.all
 });
 function createDiffString(expected, actual, diffOptions) {
   const opts = { ...diffOptions, color: false };
@@ -952,9 +981,12 @@ async function collectTestCasesFromFiles(config) {
 }
 
 // src/runner/score-utils.ts
+function getScoreDef(item) {
+  return item.def ?? getScoreById(item.id);
+}
 function toNumericScoreFromScores(scores) {
   for (const item of scores) {
-    const def = getScoreById(item.id);
+    const def = getScoreDef(item);
     if (def && def.displayStrategy === "bar" && typeof item.data === "object" && item.data !== null && "value" in item.data) {
       const value = item.data.value;
       if (typeof value === "number" && Number.isFinite(value)) {
@@ -1035,6 +1067,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
+      const evaluatorRunId = `run-${randomUUID()}`;
       const started = Date.now();
       const evaluatorScores = [];
       let testCaseError;
@@ -1061,6 +1094,11 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
+                meta: {
+                  triggerId: task.triggerId,
+                  runId: evaluatorRunId,
+                  datasetId: task.datasetId
+                },
                 logDiff,
                 log
               })
@@ -1336,7 +1374,7 @@ var createPersistenceWorker = (queue) => Effect.forever(
       () => appendJsonLine(message.artifactPath, {
         runId: message.runId,
         ts: Date.now(),
-        ...message.payload
+        ...typeof message.payload === "object" && message.payload !== null && !Array.isArray(message.payload) ? message.payload : {}
       })
     );
   })
@@ -1520,6 +1558,7 @@ var EffectRunner = class {
       (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
       0
     );
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1561,6 +1600,7 @@ var EffectRunner = class {
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
+        triggerId,
         datasetId: request.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
@@ -1634,6 +1674,6 @@ var EffectRunner = class {
   }
 };
 
-export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, deltaScore, formatScoreData, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
 //# sourceMappingURL=out.js.map
 //# sourceMappingURL=index.js.map