@m4trix/evals 0.12.0 → 0.13.0

package/dist/cli.js

@@ -6,7 +6,7 @@ import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
 import { resolve, relative, join, dirname } from 'path';
 import { diffString } from 'json-diff';
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 import { existsSync } from 'fs';
 import * as jitiModule from 'jiti';
 import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
@@ -834,6 +834,7 @@ var Metric = {
     const def = {
       id: config.id,
       name: config.name,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data) => ({ id: config.id, data })
     };
@@ -853,6 +854,7 @@ var Score = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
@@ -871,23 +873,62 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateAverage(values) {
+  if (values.length === 0) {
+    return { value: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  return { value: sum / values.length };
+}
+function aggregateAll(values) {
+  return { passed: values.length > 0 && values.every((v) => v.passed) };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
 // src/evals/metrics/standard.ts
 Metric.of({
   id: "token-count",
   name: "Tokens",
-  format: (data) => {
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
     const input = data.input ?? 0;
     const output = data.output ?? 0;
     const inputCached = data.inputCached ?? 0;
     const outputCached = data.outputCached ?? 0;
     const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
   }
 });
 Metric.of({
   id: "latency",
   name: "Latency",
-  format: (data) => `${data.ms}ms`
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
 });
 
 // src/evals/scores/standard.ts
@@ -895,13 +936,15 @@ Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.value.toFixed(2)}` : data.value.toFixed(2),
+  aggregate: aggregateAverage
 });
 Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+  format: (data, options) => options?.isAggregated ? data.passed ? "All: PASSED" : "Some: FAILED" : data.passed ? "PASSED" : "NOT PASSED",
+  aggregate: aggregateAll
 });
 function createDiffLogEntry(expected, actual, options) {
   const diff = diffString(expected, actual, { color: false });
@@ -983,7 +1026,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -1016,6 +1060,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -1250,6 +1297,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* Effect.all([
+      Ref.get(passedRef),
+      Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -1262,104 +1408,51 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
-      }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const ctx = yield* Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
-      }
-    }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
-  }
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* Ref.make(0);
+  const passedRef = yield* Ref.make(0);
+  const failedRef = yield* Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
+    Ref.get(completedRef),
+    Ref.get(passedRef),
+    Ref.get(failedRef)
+  ]);
   const finishedAt = Date.now();
   const completedEvent = {
     type: "RunCompleted",
     runId: task.runId,
     finishedAt,
-    passedTestCases,
-    failedTestCases,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
   updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     finishedAt
   }));
   yield* publishEvent(completedEvent);
@@ -1447,7 +1540,7 @@ async function parseArtifactToSnapshot(filePath, _config) {
   const artifactPath = filePath;
   const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
   const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
   const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
   const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
@@ -1469,23 +1562,29 @@ async function parseArtifactToSnapshot(filePath, _config) {
 }
 function aggregateTestCaseProgress(lines) {
   let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
   for (const line of lines) {
     try {
       const event = JSON.parse(line);
       if (event.type === "TestCaseProgress") {
         const ev = event;
         completedTestCases = ev.completedTestCases ?? completedTestCases;
-        if (ev.passed) {
-          passedTestCases += 1;
-        } else {
-          failedTestCases += 1;
-        }
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
       }
     } catch {
     }
   }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
   return { completedTestCases, passedTestCases, failedTestCases };
 }
 async function parseArtifactFile(artifactPath) {
@@ -1503,6 +1602,8 @@ async function parseArtifactFile(artifactPath) {
             testCaseName: ev.testCaseName,
             completedTestCases: ev.completedTestCases,
             totalTestCases: ev.totalTestCases,
+            rerunIndex: ev.rerunIndex,
+            rerunTotal: ev.rerunTotal,
             passed: ev.passed,
             durationMs: ev.durationMs,
             evaluatorScores: ev.evaluatorScores ?? []
@@ -1708,6 +1809,10 @@ var EffectRunner = class {
       throw new Error("No evaluators selected for run");
     }
     const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1720,7 +1825,7 @@ var EffectRunner = class {
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       completedTestCases: 0,
       passedTestCases: 0,
       failedTestCases: 0,
@@ -1734,7 +1839,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       artifactPath
     };
     await Effect.runPromise(this.publishEvent(queuedEvent));
@@ -1745,6 +1850,7 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
@@ -1752,7 +1858,8 @@ var EffectRunner = class {
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
-        snapshot
+        snapshot,
+        maxConcurrency
       })
     );
     return snapshot;
@@ -1949,6 +2056,7 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
     rows.push(/* @__PURE__ */ jsx(Text, { children: " " }, "sp6"));
     rows.push(/* @__PURE__ */ jsx(SectionHeader, { children: "Test cases" }, "tc-h"));
     for (const tc of testCases) {
+      const rerunPart = tc.rerunTotal != null && tc.rerunIndex != null ? ` (${tc.rerunIndex}/${tc.rerunTotal})` : "";
       rows.push(
         /* @__PURE__ */ jsxs(Text, { children: [
           /* @__PURE__ */ jsxs(Text, { color: "cyan", children: [
@@ -1960,12 +2068,13 @@ function buildDetailRows(run, testCases, evaluatorNameById) {
           ] }),
           " ",
           tc.testCaseName,
+          rerunPart ? /* @__PURE__ */ jsx(Text, { color: "cyan", children: rerunPart }) : null,
           /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
             " (",
             tc.durationMs,
             "ms)"
           ] })
-        ] }, `tc-${tc.testCaseId}`)
+        ] }, `tc-${tc.testCaseId}-${tc.rerunIndex ?? 0}`)
       );
       for (const item of tc.evaluatorScores) {
         const name = evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;