@m4trix/evals 0.21.0 → 0.22.0

package/dist/cli.js

@@ -6,12 +6,13 @@ import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
 import { resolve, relative, join, dirname } from 'path';
 import { LineGraph } from '@pppp606/ink-chart';
 import { randomUUID } from 'crypto';
-import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
+import { Effect, PubSub, Queue, Ref, Fiber } from 'effect';
 import { existsSync } from 'fs';
 import * as jitiModule from 'jiti';
 import { readdir, readFile, mkdir, appendFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
-import { diffString } from 'json-diff';
+import { diffLines } from 'diff';
+import stringify from 'fast-json-stable-stringify';
 
 var SEP = "  ";
 var ARROW = "\u203A";
@@ -978,10 +979,102 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify(preprocessForDiff(a, options));
+      const bStr = stringify(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
+    }
+    return filtered;
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
+  }
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
+  }
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
+  }
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
+  }
+  return lines.join("\n");
+}
 function createDiffString(expected, actual, diffOptions) {
-  const opts = { ...diffOptions, color: false };
-  const result = diffString(expected, actual, opts);
-  return typeof result === "string" ? result : "";
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(
+      extractKeys(expectedProcessed),
+      null,
+      2
+    );
+    const actualKeys = JSON.stringify(
+      extractKeys(actualProcessed),
+      null,
+      2
+    );
+    const parts2 = diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
+  }
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
+  }
+  const parts = diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter(
+      (p) => p.added === true
+    );
+    return formatDiffParts(filtered);
+  }
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
+  }
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
+  }
+  return result;
 }
 function formatLogMessage(msg) {
   if (typeof msg === "string")
@@ -1323,6 +1416,20 @@ function readOutput(testCase) {
   }
   return candidate.getOutput();
 }
+function buildEvaluationUnits(testCases) {
+  const units = [];
+  for (const testCaseItem of testCases) {
+    const rerunTotal = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
+    for (let r = 0; r < rerunTotal; r++) {
+      units.push({
+        testCaseItem,
+        rerunIndex: r + 1,
+        rerunTotal
+      });
+    }
+  }
+  return units;
+}
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -1332,157 +1439,171 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
+function processOneEvaluation(task, unit, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef, testCaseResultsRef) {
+  const { testCaseItem, rerunIndex, rerunTotal } = unit;
   return Effect.gen(function* () {
-    const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
-    const rerunPassed = [];
-    for (let r = 0; r < reruns; r++) {
-      const evaluatorRunId = `run-${randomUUID()}`;
-      const started = Date.now();
-      const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      yield* publishEvent({
-        type: "TestCaseStarted",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        startedTestCases: startedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns
-      });
-      const evaluatorScores = [];
-      let testCaseError;
-      const output = readOutput(testCaseItem.testCase);
-      for (const { id: evaluatorId, evaluator } of task.evaluators) {
-        const evaluateFn = evaluator.getEvaluateFn();
-        if (!evaluateFn) {
-          continue;
-        }
-        const logs = [];
-        const logDiff = (expected, actual, options) => {
-          logs.push(createDiffLogEntry(expected, actual, options));
-        };
-        const log = (message, options) => {
-          logs.push(createLogEntry(message, options));
-        };
-        const createError = (message, options) => {
-          const entry = createLogEntry(message, options);
-          const error = message instanceof Error ? message : new Error(entry.message);
-          error[evaluatorErrorLogEntryKey] = entry;
-          return error;
-        };
-        try {
-          const ctx = yield* Effect.promise(
-            () => Promise.resolve(evaluator.resolveContext())
-          );
-          const result = yield* Effect.promise(
-            () => Promise.resolve().then(
-              () => evaluateFn({
-                input: testCaseItem.testCase.getInput(),
-                ctx,
-                output,
-                meta: {
-                  triggerId: task.triggerId,
-                  runId: evaluatorRunId,
-                  datasetId: task.datasetId
-                },
-                logDiff,
-                log,
-                createError
-              })
-            )
-          );
-          if (result instanceof Error) {
-            const evaluatorError = result;
-            const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(result));
-            testCaseError = result.message;
-            evaluatorScores.push({
-              evaluatorId,
-              scores: [],
-              passed: false,
-              logs: logs.length > 0 ? logs : void 0
-            });
-            continue;
-          }
-          const { scores, metrics } = normalizeResult(result);
-          const passed2 = computeEvaluatorPassed(evaluator, result, scores);
-          evaluatorScores.push({
-            evaluatorId,
-            scores,
-            passed: passed2,
-            metrics,
-            logs: logs.length > 0 ? logs : void 0
-          });
-        } catch (error) {
-          if (error instanceof Error) {
-            const taggedEntry = error[evaluatorErrorLogEntryKey];
-            logs.push(taggedEntry ?? createLogEntry(error));
-          }
-          testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+    const evaluatorRunId = `run-${randomUUID()}`;
+    const started = Date.now();
+    const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
+      n + 1,
+      n + 1
+    ]);
+    yield* publishEvent({
+      type: "TestCaseStarted",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      startedTestCases: startedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal
+    });
+    const evaluatorScores = [];
+    let testCaseError;
+    const output = readOutput(testCaseItem.testCase);
+    for (const { id: evaluatorId, evaluator } of task.evaluators) {
+      const evaluateFn = evaluator.getEvaluateFn();
+      if (!evaluateFn) {
+        continue;
+      }
+      const logs = [];
+      const logDiff = (expected, actual, options) => {
+        logs.push(createDiffLogEntry(expected, actual, options));
+      };
+      const log = (message, options) => {
+        logs.push(createLogEntry(message, options));
+      };
+      const createError = (message, options) => {
+        const entry = createLogEntry(message, options);
+        const error = message instanceof Error ? message : new Error(entry.message);
+        error[evaluatorErrorLogEntryKey] = entry;
+        return error;
+      };
+      try {
+        const ctx = yield* Effect.promise(
+          () => Promise.resolve(evaluator.resolveContext())
+        );
+        const result = yield* Effect.promise(
+          () => Promise.resolve().then(
+            () => evaluateFn({
+              input: testCaseItem.testCase.getInput(),
+              ctx,
+              output,
+              meta: {
+                triggerId: task.triggerId,
+                runId: evaluatorRunId,
+                datasetId: task.datasetId
+              },
+              logDiff,
+              log,
+              createError
+            })
+          )
+        );
+        if (result instanceof Error) {
+          const evaluatorError = result;
+          const taggedEntry = evaluatorError[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(result));
+          testCaseError = result.message;
           evaluatorScores.push({
             evaluatorId,
             scores: [],
             passed: false,
             logs: logs.length > 0 ? logs : void 0
           });
+          continue;
+        }
+        const { scores, metrics } = normalizeResult(result);
+        const passed = computeEvaluatorPassed(evaluator, result, scores);
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
+      } catch (error) {
+        if (error instanceof Error) {
+          const taggedEntry = error[evaluatorErrorLogEntryKey];
+          logs.push(taggedEntry ?? createLogEntry(error));
         }
+        testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
+        evaluatorScores.push({
+          evaluatorId,
+          scores: [],
+          passed: false,
+          logs: logs.length > 0 ? logs : void 0
+        });
       }
-      const rerunPassedThis = evaluatorScores.every((s) => s.passed);
-      rerunPassed.push(rerunPassedThis);
-      const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
-        n + 1,
-        n + 1
-      ]);
-      const progressEvent = {
-        type: "TestCaseProgress",
-        runId: task.runId,
-        testCaseId: testCaseItem.id,
-        testCaseName: testCaseItem.testCase.getName(),
-        completedTestCases: completedEvaluations,
-        totalTestCases: totalEvaluations,
-        rerunIndex: r + 1,
-        rerunTotal: reruns,
-        passed: rerunPassedThis,
-        durationMs: Date.now() - started,
-        evaluatorScores,
-        output,
-        errorMessage: testCaseError
-      };
-      updateSnapshot(task.runId, (snapshot) => ({
-        ...snapshot,
-        completedTestCases: completedEvaluations
-      }));
-      yield* publishEvent(progressEvent);
-      yield* Queue.offer(persistenceQueue, {
-        runId: task.runId,
-        artifactPath: task.snapshot.artifactPath,
-        payload: progressEvent
-      });
     }
-    const testCasePassed = rerunPassed.every(Boolean);
-    if (testCasePassed) {
-      yield* Ref.update(passedRef, (n) => n + 1);
-    } else {
-      yield* Ref.update(failedRef, (n) => n + 1);
-    }
-    const [passed, failed] = yield* Effect.all([
-      Ref.get(passedRef),
-      Ref.get(failedRef)
+    const rerunPassedThis = evaluatorScores.every((s) => s.passed);
+    const completedEvaluations = yield* Ref.modify(completedRef, (n) => [
+      n + 1,
+      n + 1
     ]);
-    updateSnapshot(task.runId, (snapshot) => ({
+    const progressEvent = {
+      type: "TestCaseProgress",
+      runId: task.runId,
+      testCaseId: testCaseItem.id,
+      testCaseName: testCaseItem.testCase.getName(),
+      completedTestCases: completedEvaluations,
+      totalTestCases: totalEvaluations,
+      rerunIndex,
+      rerunTotal,
+      passed: rerunPassedThis,
+      durationMs: Date.now() - started,
+      evaluatorScores,
+      output,
+      errorMessage: testCaseError
+    };
+    yield* updateSnapshot(task.runId, (snapshot) => ({
       ...snapshot,
-      passedTestCases: passed,
-      failedTestCases: failed
+      completedTestCases: completedEvaluations
     }));
+    yield* publishEvent(progressEvent);
+    yield* Queue.offer(persistenceQueue, {
+      runId: task.runId,
+      artifactPath: task.snapshot.artifactPath,
+      payload: progressEvent
+    });
+    const testCaseCompleted = yield* Ref.modify(
+      testCaseResultsRef,
+      (map) => {
+        const key = testCaseItem.id;
+        const existing = map.get(key) ?? { completedCount: 0, results: [] };
+        const newResults = [...existing.results, rerunPassedThis];
+        const newCompletedCount = existing.completedCount + 1;
+        const isLast = newCompletedCount === rerunTotal;
+        const newMap = new Map(map);
+        newMap.set(key, {
+          completedCount: newCompletedCount,
+          results: newResults
+        });
+        const outcome = isLast ? newResults.every(Boolean) : null;
+        return [outcome, newMap];
+      }
+    );
+    if (testCaseCompleted !== null) {
+      if (testCaseCompleted) {
+        yield* Ref.update(passedRef, (n) => n + 1);
+      } else {
+        yield* Ref.update(failedRef, (n) => n + 1);
+      }
+      const [passed, failed] = yield* Effect.all([
+        Ref.get(passedRef),
+        Ref.get(failedRef)
+      ]);
+      yield* updateSnapshot(task.runId, (snapshot) => ({
+        ...snapshot,
+        passedTestCases: passed,
+        failedTestCases: failed
+      }));
+    }
   });
 }
 var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => Effect.gen(function* () {
   const startedAt = Date.now();
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "running",
     startedAt
@@ -1501,9 +1622,13 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   const startedRef = yield* Ref.make(0);
   const passedRef = yield* Ref.make(0);
   const failedRef = yield* Ref.make(0);
-  const processTestCase = (testCaseItem) => processOneTestCase(
+  const testCaseResultsRef = yield* Ref.make(
+    /* @__PURE__ */ new Map()
+  );
+  const evaluationUnits = buildEvaluationUnits(task.testCases);
+  const processEvaluation = (unit) => processOneEvaluation(
     task,
-    testCaseItem,
+    unit,
     totalEvaluations,
     publishEvent,
     persistenceQueue,
@@ -1511,11 +1636,12 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     startedRef,
     completedRef,
     passedRef,
-    failedRef
+    failedRef,
+    testCaseResultsRef
   );
   yield* Effect.forEach(
-    task.testCases,
-    processTestCase,
+    evaluationUnits,
+    processEvaluation,
     maxConcurrency > 1 ? { concurrency: maxConcurrency } : void 0
   );
   const [completedEvaluations, passedUniqueTestCases, failedUniqueTestCases] = yield* Effect.all([
@@ -1533,7 +1659,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     totalTestCases: task.testCases.length,
     artifactPath: task.snapshot.artifactPath
   };
-  updateSnapshot(task.runId, (snapshot) => ({
+  yield* updateSnapshot(task.runId, (snapshot) => ({
     ...snapshot,
     status: "completed",
     completedTestCases: completedEvaluations,
@@ -1816,7 +1942,9 @@ var EffectRunner = class {
     this.persistenceQueue = Effect.runSync(
       Queue.unbounded()
     );
-    this.snapshots = /* @__PURE__ */ new Map();
+    this.snapshotsRef = Effect.runSync(
+      Ref.make(/* @__PURE__ */ new Map())
+    );
     this.listeners = /* @__PURE__ */ new Set();
     this.datasetsById = /* @__PURE__ */ new Map();
     this.evaluatorsById = /* @__PURE__ */ new Map();
@@ -1919,7 +2047,13 @@ var EffectRunner = class {
       status: "queued",
       artifactPath
     };
-    this.snapshots.set(runId, snapshot);
+    await Effect.runPromise(
+      Ref.update(this.snapshotsRef, (map) => {
+        const next = new Map(map);
+        next.set(runId, snapshot);
+        return next;
+      })
+    );
     const queuedEvent = {
       type: "RunQueued",
       runId,
@@ -1960,12 +2094,12 @@ var EffectRunner = class {
     };
   }
   getRunSnapshot(runId) {
-    return this.snapshots.get(runId);
+    return Effect.runSync(Ref.get(this.snapshotsRef)).get(runId);
   }
   getAllRunSnapshots() {
-    return Array.from(this.snapshots.values()).sort(
-      (a, b) => b.queuedAt - a.queuedAt
-    );
+    return Array.from(
+      Effect.runSync(Ref.get(this.snapshotsRef)).values()
+    ).sort((a, b) => b.queuedAt - a.queuedAt);
   }
   async loadRunSnapshotsFromArtifacts() {
     return loadRunSnapshotsFromArtifacts(this.config);
@@ -1994,11 +2128,15 @@ var EffectRunner = class {
     );
   }
   updateSnapshot(runId, updater) {
-    const existing = this.snapshots.get(runId);
-    if (!existing) {
-      return;
-    }
-    this.snapshots.set(runId, updater(existing));
+    return Ref.modify(this.snapshotsRef, (map) => {
+      const existing = map.get(runId);
+      if (!existing) {
+        return [void 0, map];
+      }
+      const next = new Map(map);
+      next.set(runId, updater(existing));
+      return [void 0, next];
+    }).pipe(Effect.asVoid);
   }
   publishEvent(event) {
     return Effect.sync(() => {