@m4trix/evals 0.12.0 → 0.13.0

package/dist/index.d.ts

@@ -91,6 +91,8 @@ interface RunnerDiscoveryConfig {
 interface RunnerConfig {
     discovery: RunnerDiscoveryConfig;
     artifactDirectory: string;
+    /** Max concurrent test cases per run. Default: 1 (sequential). */
+    maxConcurrency: number;
 }
 type RunnerConfigOverrides = Omit<Partial<RunnerConfig>, 'discovery'> & {
     discovery?: Partial<RunnerDiscoveryConfig>;
@@ -108,6 +110,8 @@ interface M4trixEvalConfigDiscovery {
 interface M4trixEvalConfig {
     discovery?: M4trixEvalConfigDiscovery;
     artifactDirectory?: string;
+    /** Max concurrent test cases per run. Default: 1 (sequential). */
+    maxConcurrency?: number;
 }
 type ConfigType = M4trixEvalConfig;
 type M4trixEvalConfigFactory<TConfig extends ConfigType = ConfigType> = () => TConfig;
@@ -124,6 +128,7 @@ type InputOrBuilder<T> = T | (() => T);
 interface TestCaseDescribeConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>> {
     name: string;
     tags: string[];
+    reruns?: number;
     inputSchema: TI;
     input: InputOrBuilder<Schema.Schema.Type<TI>>;
     outputSchema?: TO;
@@ -133,6 +138,7 @@ declare class TestCase<TInput = unknown, TOutput = unknown> {
     private readonly _config;
     private constructor();
     static describe<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>>(config: TestCaseDescribeConfig<TI, TO>): TestCase<Schema.Schema.Type<TI>, Schema.Schema.Type<TO>>;
+    getReruns(): number;
     getName(): string;
     getTags(): string[];
     getInputSchema(): Schema.Schema.Any;
@@ -225,17 +231,22 @@ interface MetricItem<TData = unknown> {
     readonly id: string;
     readonly data: TData;
 }
+interface FormatMetricOptions {
+    isAggregated?: boolean;
+}
 interface MetricDef<TData = unknown> {
     readonly id: string;
     readonly name?: string;
-    format(data: TData): string;
+    readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
+    format(data: TData, options?: FormatMetricOptions): string;
     make(data: TData): MetricItem<TData>;
 }
 declare const Metric: {
     of<TData>(config: {
         id: string;
         name?: string | undefined;
-        format: (data: TData) => string;
+        format: (data: TData, options?: FormatMetricOptions) => string;
+        aggregate?: ((values: readonly TData[]) => TData) | undefined;
     }): MetricDef<TData>;
 };
 declare function getMetricById(id: string): MetricDef<unknown> | undefined;
@@ -246,11 +257,15 @@ interface ScoreItem<TData = unknown> {
     readonly data: TData;
     readonly passed?: boolean;
 }
+interface FormatScoreOptions {
+    isAggregated?: boolean;
+}
 interface ScoreDef<TData = unknown> {
     readonly id: string;
     readonly name?: string;
     readonly displayStrategy: ScoreDisplayStrategy;
-    format(data: TData): string;
+    readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
+    format(data: TData, options?: FormatScoreOptions): string;
     make(data: TData, options?: {
         definePassed?: (data: TData) => boolean;
     }): ScoreItem<TData>;
@@ -260,7 +275,8 @@ declare const Score: {
         id: string;
         name?: string | undefined;
         displayStrategy: ScoreDisplayStrategy;
-        format: (data: TData) => string;
+        format: (data: TData, options?: FormatScoreOptions) => string;
+        aggregate?: ((values: readonly TData[]) => TData) | undefined;
     }): ScoreDef<TData>;
 };
 declare function getScoreById(id: string): ScoreDef<unknown> | undefined;
@@ -326,6 +342,8 @@ type RunnerEvent = {
     testCaseName: string;
     completedTestCases: number;
     totalTestCases: number;
+    rerunIndex: number;
+    rerunTotal: number;
     passed: boolean;
     durationMs: number;
     evaluatorScores: ReadonlyArray<{
@@ -401,4 +419,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 export { Schema as S } from 'effect';
 import { diffString } from 'json-diff';
 import { randomUUID } from 'crypto';
@@ -309,15 +309,23 @@ var TestCase = class _TestCase {
     this._config = config;
   }
   static describe(config) {
+    const reruns = config.reruns ?? 1;
+    if (reruns < 1 || !Number.isInteger(reruns)) {
+      throw new Error(`TestCase reruns must be a positive integer, got ${reruns}`);
+    }
     return new _TestCase({
       name: config.name,
       tags: config.tags,
+      reruns,
       inputSchema: config.inputSchema,
       input: config.input,
       outputSchema: config.outputSchema,
       output: config.output
     });
   }
+  getReruns() {
+    return this._config.reruns;
+  }
   getName() {
     return this._config.name;
   }
@@ -491,6 +499,7 @@ var Metric = {
     const def = {
       id: config.id,
       name: config.name,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data) => ({ id: config.id, data })
     };
@@ -510,6 +519,7 @@ var Score = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
@@ -528,23 +538,62 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateAverage(values) {
+  if (values.length === 0) {
+    return { value: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  return { value: sum / values.length };
+}
+function aggregateAll(values) {
+  return { passed: values.length > 0 && values.every((v) => v.passed) };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
 // src/evals/metrics/standard.ts
 var tokenCountMetric = Metric.of({
   id: "token-count",
   name: "Tokens",
-  format: (data) => {
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
     const input = data.input ?? 0;
     const output = data.output ?? 0;
     const inputCached = data.inputCached ?? 0;
     const outputCached = data.outputCached ?? 0;
     const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
   }
 });
 var latencyMetric = Metric.of({
   id: "latency",
   name: "Latency",
-  format: (data) => `${data.ms}ms`
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
 });
 
 // src/evals/scores/standard.ts
@@ -552,13 +601,15 @@ var percentScore = Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
+  aggregate: aggregateAverage
 });
 var binaryScore = Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  aggregate: aggregateAll
 });
 function createDiffLogEntry(expected, actual, options) {
   const diff = diffString(expected, actual, { color: false });
@@ -599,7 +650,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -632,6 +684,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -905,6 +960,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* Effect.all([
+      Ref.get(passedRef),
+      Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -917,104 +1071,51 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
-      }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const ctx = yield* Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
-      }
-    }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
-  }
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* Ref.make(0);
+  const passedRef = yield* Ref.make(0);
+  const failedRef = yield* Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
+    Ref.get(completedRef),
+    Ref.get(passedRef),
+    Ref.get(failedRef)
+  ]);
   const finishedAt = Date.now();
   const completedEvent = {
     type: "RunCompleted",
     runId: task.runId,
     finishedAt,
-    passedTestCases,
-    failedTestCases,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
   updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     finishedAt
   }));
   yield* publishEvent(completedEvent);
@@ -1102,7 +1203,7 @@ async function parseArtifactToSnapshot(filePath, _config) {
   const artifactPath = filePath;
   const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
   const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
   const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
   const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
@@ -1124,23 +1225,29 @@ async function parseArtifactToSnapshot(filePath, _config) {
 }
 function aggregateTestCaseProgress(lines) {
   let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
   for (const line of lines) {
     try {
       const event = JSON.parse(line);
       if (event.type === "TestCaseProgress") {
         const ev = event;
         completedTestCases = ev.completedTestCases ?? completedTestCases;
-        if (ev.passed) {
-          passedTestCases += 1;
-        } else {
-          failedTestCases += 1;
-        }
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
       }
     } catch {
     }
   }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
   return { completedTestCases, passedTestCases, failedTestCases };
 }
 async function appendJsonLine(artifactPath, payload) {
@@ -1335,6 +1442,10 @@ var EffectRunner = class {
       throw new Error("No evaluators selected for run");
     }
     const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1347,7 +1458,7 @@ var EffectRunner = class {
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       completedTestCases: 0,
       passedTestCases: 0,
       failedTestCases: 0,
@@ -1361,7 +1472,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       artifactPath
     };
     await Effect.runPromise(this.publishEvent(queuedEvent));
@@ -1372,6 +1483,7 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
@@ -1379,7 +1491,8 @@ var EffectRunner = class {
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
-        snapshot
+        snapshot,
+        maxConcurrency
       })
     );
     return snapshot;