@m4trix/evals 0.12.0 → 0.14.0

package/dist/index.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
 var effect = require('effect');
-var jsonDiff = require('json-diff');
+var diff = require('diff');
 var crypto = require('crypto');
 var fs = require('fs');
 var path = require('path');
@@ -331,15 +331,23 @@ var TestCase = class _TestCase {
     this._config = config;
   }
   static describe(config) {
+    const reruns = config.reruns ?? 1;
+    if (reruns < 1 || !Number.isInteger(reruns)) {
+      throw new Error(`TestCase reruns must be a positive integer, got ${reruns}`);
+    }
     return new _TestCase({
       name: config.name,
       tags: config.tags,
+      reruns,
       inputSchema: config.inputSchema,
       input: config.input,
       outputSchema: config.outputSchema,
       output: config.output
     });
   }
+  getReruns() {
+    return this._config.reruns;
+  }
   getName() {
     return this._config.name;
   }
@@ -513,6 +521,7 @@ var Metric = {
     const def = {
       id: config.id,
       name: config.name,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data) => ({ id: config.id, data })
     };
@@ -532,6 +541,7 @@ var Score = {
       id: config.id,
       name: config.name,
       displayStrategy: config.displayStrategy,
+      aggregate: config.aggregate,
       format: config.format,
       make: (data, options) => {
         const passed = options?.definePassed !== void 0 ? options.definePassed(data) : void 0;
@@ -550,23 +560,75 @@ function getScoreById(id) {
   return registry2.get(id);
 }
 
+// src/evals/aggregators.ts
+function aggregateAverageWithVariance(values) {
+  if (values.length === 0) {
+    return { value: 0, count: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.value, 0);
+  const sumSq = values.reduce((s, v) => s + v.value * v.value, 0);
+  const mean = sum / values.length;
+  let stdDev;
+  if (values.length >= 2) {
+    const variance = (sumSq - values.length * mean * mean) / (values.length - 1);
+    stdDev = variance > 0 ? Math.sqrt(variance) : 0;
+  }
+  return { value: mean, stdDev, count: values.length };
+}
+function aggregateAll(values) {
+  const total = values.length;
+  const passedCount = values.filter((v) => v.passed).length;
+  return {
+    passed: total > 0 && values.every((v) => v.passed),
+    passedCount,
+    totalCount: total
+  };
+}
+function aggregateTokenCountSum(values) {
+  const initial = {
+    input: 0,
+    output: 0,
+    inputCached: 0,
+    outputCached: 0
+  };
+  return values.reduce(
+    (acc, v) => ({
+      input: acc.input + (v.input ?? 0),
+      output: acc.output + (v.output ?? 0),
+      inputCached: acc.inputCached + (v.inputCached ?? 0),
+      outputCached: acc.outputCached + (v.outputCached ?? 0)
+    }),
+    initial
+  );
+}
+function aggregateLatencyAverage(values) {
+  if (values.length === 0) {
+    return { ms: 0 };
+  }
+  const sum = values.reduce((s, v) => s + v.ms, 0);
+  return { ms: sum / values.length };
+}
+
 // src/evals/metrics/standard.ts
 var tokenCountMetric = Metric.of({
   id: "token-count",
   name: "Tokens",
-  format: (data) => {
+  aggregate: aggregateTokenCountSum,
+  format: (data, options) => {
     const input = data.input ?? 0;
     const output = data.output ?? 0;
     const inputCached = data.inputCached ?? 0;
     const outputCached = data.outputCached ?? 0;
     const cached = inputCached + outputCached;
-    return `in:${input} out:${output} cached:${cached}`;
+    const base = `in:${input} out:${output} cached:${cached}`;
+    return options?.isAggregated ? `Total: ${base}` : base;
   }
 });
 var latencyMetric = Metric.of({
   id: "latency",
   name: "Latency",
-  format: (data) => `${data.ms}ms`
+  aggregate: aggregateLatencyAverage,
+  format: (data, options) => options?.isAggregated ? `Avg: ${data.ms}ms` : `${data.ms}ms`
 });
 
 // src/evals/scores/standard.ts
@@ -574,16 +636,59 @@ var percentScore = Score.of({
   id: "percent",
   name: "Score",
   displayStrategy: "bar",
-  format: (data) => data.value.toFixed(2)
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      return data.stdDev != null ? `Avg: ${data.value.toFixed(2)} \xB1 ${data.stdDev.toFixed(2)}` : `Avg: ${data.value.toFixed(2)}`;
+    }
+    return data.value.toFixed(2);
+  },
+  aggregate: aggregateAverageWithVariance
 });
 var binaryScore = Score.of({
   id: "binary",
   name: "Result",
   displayStrategy: "passFail",
-  format: (data) => data.passed ? "PASSED" : "NOT PASSED"
+  format: (data, options) => {
+    if (options?.isAggregated) {
+      const base = data.passed ? "All: PASSED" : "Some: FAILED";
+      if (data.passedCount != null && data.totalCount != null && data.totalCount > 1) {
+        return `${base} (${data.passedCount}/${data.totalCount})`;
+      }
+      return base;
+    }
+    return data.passed ? "PASSED" : "NOT PASSED";
+  },
+  aggregate: aggregateAll
 });
+function toJsonLines(value) {
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return String(value);
+  }
+}
+function formatDiffString(changes) {
+  const lines = [];
+  for (const part of changes) {
+    const prefix = part.added ? "+" : part.removed ? "-" : " ";
+    const partLines = part.value.split("\n");
+    if (partLines[partLines.length - 1] === "") {
+      partLines.pop();
+    }
+    for (const line of partLines) {
+      lines.push(`${prefix} ${line}`);
+    }
+  }
+  return lines.join("\n");
+}
+function createDiffString(expected, actual) {
+  const expectedStr = toJsonLines(expected);
+  const actualStr = toJsonLines(actual);
+  const changes = diff.diffLines(expectedStr, actualStr);
+  return formatDiffString(changes);
+}
 function createDiffLogEntry(expected, actual, options) {
-  const diff = jsonDiff.diffString(expected, actual, { color: false });
+  const diff = createDiffString(expected, actual);
   return {
     type: "diff",
     label: options?.label,
@@ -593,8 +698,22 @@ function createDiffLogEntry(expected, actual, options) {
   };
 }
 function printJsonDiff(expected, actual, options = {}) {
-  const { color = true } = options;
-  const diff = jsonDiff.diffString(expected, actual, { color });
+  const diff = createDiffString(expected, actual);
+  if (options.color) {
+    const lines = diff.split("\n").map((line) => {
+      const trimmed = line.trimStart();
+      if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+        return `\x1B[31m${line}\x1B[0m`;
+      }
+      if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+        return `\x1B[32m${line}\x1B[0m`;
+      }
+      return line;
+    });
+    const colored = lines.join("\n");
+    console.log(colored || "(no differences)");
+    return colored;
+  }
   console.log(diff || "(no differences)");
   return diff;
 }
@@ -621,7 +740,8 @@ var defaultRunnerConfig = {
     ],
     excludeDirectories: ["node_modules", "dist", ".next", ".git", ".pnpm-store"]
   },
-  artifactDirectory: ".eval-results"
+  artifactDirectory: ".eval-results",
+  maxConcurrency: 1
 };
 function toRunnerConfigOverrides(config) {
   if (!config) {
@@ -654,6 +774,9 @@ function toRunnerConfigOverrides(config) {
   if (config.artifactDirectory !== void 0) {
     overrides.artifactDirectory = config.artifactDirectory;
   }
+  if (config.maxConcurrency !== void 0) {
+    overrides.maxConcurrency = config.maxConcurrency;
+  }
   if (Object.keys(discovery).length > 0) {
     overrides.discovery = discovery;
   }
@@ -927,6 +1050,105 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+  return effect.Effect.gen(function* () {
+    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    const rerunPassed = [];
+    for (let r = 0; r < reruns; r++) {
+      const started = Date.now();
+      const evaluatorScores = [];
+      let testCaseError;
+      const output = readOutput(testCaseItem.testCase);
+      for (const { id: evaluatorId, evaluator } of task.evaluators) {
+        const evaluateFn = evaluator.getEvaluateFn();
+        if (!evaluateFn) {
+          continue;
+        }
+        try {
+          const logs = [];
+          const logDiff = (expected, actual, options) => {
+            logs.push(createDiffLogEntry(expected, actual, options));
+          };
+          const ctx = yield* effect.Effect.promise(
+            () => Promise.resolve(evaluator.resolveContext())
+          );
+          const result = yield* effect.Effect.promise(
+            () => Promise.resolve(
+              evaluateFn({
+                input: testCaseItem.testCase.getInput(),
+                ctx,
+                output,
+                logDiff
+              })
+            )
+          );
+          const { scores, metrics } = normalizeResult(result);
+          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
+          evaluatorScores.push({
+            evaluatorId,
+            scores,
+            passed: passed2,
+            metrics,
+            logs: logs.length > 0 ? logs : void 0
+          });
+        } catch (error) {
+          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+          evaluatorScores.push({
+            evaluatorId,
+            scores: [],
+            passed: false
+          });
+        }
+      }
+      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+      rerunPassed.push(rerunPassedThis);
+      const completedEvaluations = yield* effect.Ref.modify(completedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      const progressEvent = {
+        type: "TestCaseProgress",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        completedTestCases: completedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns,
+        passed: rerunPassedThis,
+        durationMs: Date.now() - started,
+        evaluatorScores,
+        output,
+        errorMessage: testCaseError
+      };
+      updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        completedTestCases: completedEvaluations
+      }));
+      yield* publishEvent(progressEvent);
+      yield* effect.Queue.offer(persistenceQueue, {
+        runId: task.runId,
+        artifactPath: task.snapshot.artifactPath,
+        payload: progressEvent
+      });
+    }
+    const testCasePassed = rerunPassed.every(Boolean);
+    if (testCasePassed) {
+      yield* effect.Ref.update(passedRef, (n) => n + 1);
+    } else {
+      yield* effect.Ref.update(failedRef, (n) => n + 1);
+    }
+    const [passed, failed] = yield* effect.Effect.all([
+      effect.Ref.get(passedRef),
+      effect.Ref.get(failedRef)
+    ]);
+    updateSnapshot(task.runId, (snapshot) => ({
+      ...snapshot,
+      passedTestCases: passed,
+      failedTestCases: failed
+    }));
+  });
+}
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => effect.Effect.gen(function* () {
   const startedAt = Date.now();
   updateSnapshot(task.runId, (snapshot) => ({
@@ -939,104 +1161,51 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     runId: task.runId,
     startedAt
   });
-  let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
-  for (const testCaseItem of task.testCases) {
-    const started = Date.now();
-    const evaluatorScores = [];
-    let testCaseError;
-    const output = readOutput(testCaseItem.testCase);
-    for (const { id: evaluatorId, evaluator } of task.evaluators) {
-      const evaluateFn = evaluator.getEvaluateFn();
-      if (!evaluateFn) {
-        continue;
-      }
-      try {
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const ctx = yield* effect.Effect.promise(
-          () => Promise.resolve(evaluator.resolveContext())
-        );
-        const result = yield* effect.Effect.promise(
-          () => Promise.resolve(
-            evaluateFn({
-              input: testCaseItem.testCase.getInput(),
-              ctx,
-              output,
-              logDiff
-            })
-          )
-        );
-        const { scores, metrics } = normalizeResult(result);
-        const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({
-          evaluatorId,
-          scores,
-          passed,
-          metrics,
-          logs: logs.length > 0 ? logs : void 0
-        });
-      } catch (error) {
-        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
-        evaluatorScores.push({
-          evaluatorId,
-          scores: [],
-          passed: false
-        });
-      }
-    }
-    const testCasePassed = evaluatorScores.every((s) => s.passed);
-    completedTestCases += 1;
-    if (testCasePassed) {
-      passedTestCases += 1;
-    } else {
-      failedTestCases += 1;
-    }
-    const progressEvent = {
-      type: "TestCaseProgress",
-      runId: task.runId,
-      testCaseId: testCaseItem.id,
-      testCaseName: testCaseItem.testCase.getName(),
-      completedTestCases,
-      totalTestCases: task.testCases.length,
-      passed: testCasePassed,
-      durationMs: Date.now() - started,
-      evaluatorScores,
-      output,
-      errorMessage: testCaseError
-    };
-    updateSnapshot(task.runId, (snapshot) => ({
-      ...snapshot,
-      completedTestCases,
-      passedTestCases,
-      failedTestCases
-    }));
-    yield* publishEvent(progressEvent);
-    yield* effect.Queue.offer(persistenceQueue, {
-      runId: task.runId,
-      artifactPath: task.snapshot.artifactPath,
-      payload: progressEvent
-    });
-  }
+  const totalEvaluations = task.testCases.reduce(
+    (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+    0
+  );
+  const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
+  const completedRef = yield* effect.Ref.make(0);
+  const passedRef = yield* effect.Ref.make(0);
+  const failedRef = yield* effect.Ref.make(0);
+  const processTestCase = (testCaseItem) => processOneTestCase(
+    task,
+    testCaseItem,
+    totalEvaluations,
+    publishEvent,
+    persistenceQueue,
+    updateSnapshot,
+    completedRef,
+    passedRef,
+    failedRef
+  );
+  yield* effect.Effect.forEach(
+    task.testCases,
+    processTestCase,
+    maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
+  );
+  const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* effect.Effect.all([
+    effect.Ref.get(completedRef),
+    effect.Ref.get(passedRef),
+    effect.Ref.get(failedRef)
+  ]);
   const finishedAt = Date.now();
   const completedEvent = {
     type: "RunCompleted",
     runId: task.runId,
     finishedAt,
-    passedTestCases,
-    failedTestCases,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
   updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
-    completedTestCases,
-    passedTestCases,
-    failedTestCases,
+    completedTestCases: completedEvaluations,
+    passedTestCases: passedUniqueTestCases,
+    failedTestCases: failedUniqueTestCases,
     finishedAt
   }));
   yield* publishEvent(completedEvent);
@@ -1124,7 +1293,7 @@ async function parseArtifactToSnapshot(filePath, _config) {
   const artifactPath = filePath;
   const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
   const progress = aggregateTestCaseProgress(lines);
-  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const completedTestCases = runCompleted ? runQueued.totalTestCases : progress.completedTestCases;
   const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
   const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
   return {
@@ -1146,23 +1315,29 @@ async function parseArtifactToSnapshot(filePath, _config) {
 }
 function aggregateTestCaseProgress(lines) {
   let completedTestCases = 0;
-  let passedTestCases = 0;
-  let failedTestCases = 0;
+  const testCasePassedBy = /* @__PURE__ */ new Map();
   for (const line of lines) {
     try {
       const event = JSON.parse(line);
       if (event.type === "TestCaseProgress") {
         const ev = event;
         completedTestCases = ev.completedTestCases ?? completedTestCases;
-        if (ev.passed) {
-          passedTestCases += 1;
-        } else {
-          failedTestCases += 1;
-        }
+        const id = ev.testCaseId;
+        const current = testCasePassedBy.get(id);
+        testCasePassedBy.set(id, current === void 0 ? ev.passed : current && ev.passed);
       }
     } catch {
     }
   }
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const passed of testCasePassedBy.values()) {
+    if (passed) {
+      passedTestCases += 1;
+    } else {
+      failedTestCases += 1;
+    }
+  }
   return { completedTestCases, passedTestCases, failedTestCases };
 }
 async function appendJsonLine(artifactPath, payload) {
@@ -1357,6 +1532,10 @@ var EffectRunner = class {
       throw new Error("No evaluators selected for run");
     }
     const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);
+    const totalEvaluations = selectedTestCases.reduce(
+      (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
+      0
+    );
     const runId = `run-${crypto.randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1369,7 +1548,7 @@ var EffectRunner = class {
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
       queuedAt: Date.now(),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       completedTestCases: 0,
       passedTestCases: 0,
       failedTestCases: 0,
@@ -1383,7 +1562,7 @@ var EffectRunner = class {
       datasetId: request.datasetId,
       datasetName: dataset.dataset.getName(),
       evaluatorIds: selectedEvaluators.map((item) => item.id),
-      totalTestCases: selectedTestCases.length,
+      totalTestCases: totalEvaluations,
       artifactPath
     };
     await effect.Effect.runPromise(this.publishEvent(queuedEvent));
@@ -1394,6 +1573,7 @@ var EffectRunner = class {
         payload: queuedEvent
       })
     );
+    const maxConcurrency = request.concurrency ?? this.config.maxConcurrency ?? 1;
     await effect.Effect.runPromise(
       effect.Queue.offer(this.runQueue, {
         runId,
@@ -1401,7 +1581,8 @@ var EffectRunner = class {
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
         testCases: selectedTestCases,
-        snapshot
+        snapshot,
+        maxConcurrency
       })
     );
     return snapshot;