@m4trix/evals 0.10.0 → 0.12.0

package/dist/index.cjs

@@ -279,11 +279,17 @@ function toEvaluatorOption(item) {
   };
 }
 async function loadRunnerData(runner) {
-  const [datasets, evaluators] = await Promise.all([
+  const [datasets, evaluators, diskSnapshots] = await Promise.all([
     runner.collectDatasets(),
-    runner.collectEvaluators()
+    runner.collectEvaluators(),
+    runner.loadRunSnapshotsFromArtifacts()
   ]);
-  const snapshots = runner.getAllRunSnapshots();
+  const memSnapshots = runner.getAllRunSnapshots();
+  const seen = new Set(memSnapshots.map((s) => s.runId));
+  const fromDisk = diskSnapshots.filter((s) => !seen.has(s.runId));
+  const snapshots = [...memSnapshots, ...fromDisk].sort(
+    (a, b) => b.queuedAt - a.queuedAt
+  );
   if (datasets.length === 0 && evaluators.length === 0) {
     return loadMockData();
   }
@@ -576,6 +582,16 @@ var binaryScore = Score.of({
   displayStrategy: "passFail",
   format: (data) => data.passed ? "PASSED" : "NOT PASSED"
 });
+function createDiffLogEntry(expected, actual, options) {
+  const diff = jsonDiff.diffString(expected, actual, { color: false });
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
 function printJsonDiff(expected, actual, options = {}) {
   const { color = true } = options;
   const diff = jsonDiff.diffString(expected, actual, { color });
@@ -937,6 +953,10 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
         continue;
       }
       try {
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
         const ctx = yield* effect.Effect.promise(
           () => Promise.resolve(evaluator.resolveContext())
         );
@@ -945,13 +965,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output
+              output,
+              logDiff
             })
           )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
       } catch (error) {
         testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
         evaluatorScores.push({
@@ -1024,6 +1051,120 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     artifactPath: task.snapshot.artifactPath
   });
 });
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = path.resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await promises.readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = path.join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await promises.readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        if (ev.passed) {
+          passedTestCases += 1;
+        } else {
+          failedTestCases += 1;
+        }
+      }
+    } catch {
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
 async function appendJsonLine(artifactPath, payload) {
   await promises.mkdir(path.dirname(artifactPath), { recursive: true });
   await promises.appendFile(artifactPath, `${JSON.stringify(payload)}
@@ -1280,6 +1421,9 @@ var EffectRunner = class {
       (a, b) => b.queuedAt - a.queuedAt
     );
   }
+  async loadRunSnapshotsFromArtifacts() {
+    return loadRunSnapshotsFromArtifacts(this.config);
+  }
   async shutdown() {
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.schedulerFiber));
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.persistenceFiber));