@m4trix/evals 0.10.0 → 0.12.0

package/dist/cli-simple.cjs

@@ -8,7 +8,7 @@ var path = require('path');
 var jitiModule = require('jiti');
 var promises = require('fs/promises');
 var url = require('url');
-require('json-diff');
+var jsonDiff = require('json-diff');
 var React2 = require('react');
 var ink = require('ink');
 var jsxRuntime = require('react/jsx-runtime');
@@ -282,6 +282,29 @@ async function collectTestCasesFromFiles(config) {
   );
   return found.flat();
 }
+function createDiffLogEntry(expected, actual, options) {
+  const diff = jsonDiff.diffString(expected, actual, { color: false });
+  return {
+    type: "diff",
+    label: options?.label,
+    expected,
+    actual,
+    diff: diff || "(no differences)"
+  };
+}
+function getDiffLines(entry) {
+  const raw = jsonDiff.diffString(entry.expected, entry.actual, { color: false }) || "(no differences)";
+  return raw.split("\n").map((line) => {
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
+      return { type: "remove", line };
+    }
+    if (trimmed.startsWith("+") && !trimmed.startsWith("+++")) {
+      return { type: "add", line };
+    }
+    return { type: "context", line };
+  });
+}
 
 // src/evals/metric.ts
 var registry = /* @__PURE__ */ new Map();
@@ -465,6 +488,10 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
         continue;
       }
       try {
+        const logs = [];
+        const logDiff = (expected, actual, options) => {
+          logs.push(createDiffLogEntry(expected, actual, options));
+        };
         const ctx = yield* effect.Effect.promise(
           () => Promise.resolve(evaluator.resolveContext())
         );
@@ -473,13 +500,20 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output
+              output,
+              logDiff
             })
           )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
-        evaluatorScores.push({ evaluatorId, scores, passed, metrics });
+        evaluatorScores.push({
+          evaluatorId,
+          scores,
+          passed,
+          metrics,
+          logs: logs.length > 0 ? logs : void 0
+        });
       } catch (error) {
         testCaseError = error instanceof Error ? error.message : "Evaluator execution failed";
         evaluatorScores.push({
@@ -552,6 +586,120 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     artifactPath: task.snapshot.artifactPath
   });
 });
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = path.resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await promises.readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = path.join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await promises.readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        if (ev.passed) {
+          passedTestCases += 1;
+        } else {
+          failedTestCases += 1;
+        }
+      }
+    } catch {
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
 async function appendJsonLine(artifactPath, payload) {
   await promises.mkdir(path.dirname(artifactPath), { recursive: true });
   await promises.appendFile(artifactPath, `${JSON.stringify(payload)}
@@ -808,6 +956,9 @@ var EffectRunner = class {
       (a, b) => b.queuedAt - a.queuedAt
     );
   }
+  async loadRunSnapshotsFromArtifacts() {
+    return loadRunSnapshotsFromArtifacts(this.config);
+  }
   async shutdown() {
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.schedulerFiber));
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.persistenceFiber));
@@ -939,7 +1090,7 @@ function GenerateView({
         return;
       }
       const { writeFile: writeFile2 } = await import('fs/promises');
-      const { join: join3, parse: parse2, resolve: resolve4 } = await import('path');
+      const { join: join4, parse: parse2, resolve: resolve5 } = await import('path');
       const testCases = await runner.collectDatasetTestCases(dataset.id);
       const payload = testCases.map((item) => {
         const tc = item.testCase;
@@ -949,9 +1100,9 @@ function GenerateView({
           output: typeof tc.getOutput === "function" ? tc.getOutput() : void 0
         };
       });
-      const absoluteDatasetPath = resolve4(process.cwd(), dataset.filePath);
+      const absoluteDatasetPath = resolve5(process.cwd(), dataset.filePath);
       const parsed = parse2(absoluteDatasetPath);
-      const outputPath = join3(parsed.dir, `${parsed.name}.cases.json`);
+      const outputPath = join4(parsed.dir, `${parsed.name}.cases.json`);
       await writeFile2(
         outputPath,
         `${JSON.stringify(payload, null, 2)}
@@ -1026,7 +1177,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   console.log(`Wrote ${outputPath}`);
 }
 async function generateDatasetJsonCommandInk(runner, datasetName) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(GenerateView, {
         runner,
@@ -1036,7 +1187,7 @@ async function generateDatasetJsonCommandInk(runner, datasetName) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })
@@ -1164,7 +1315,7 @@ function RunView({
     const aggregates = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
     let overallScoreCount = 0;
-    const done = new Promise((resolve4) => {
+    const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
         if (event.type === "TestCaseProgress") {
           const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
@@ -1202,14 +1353,15 @@ function RunView({
                 evaluatorName: nameById.get(item.evaluatorId) ?? item.evaluatorId,
                 scores: item.scores,
                 passed: item.passed,
-                metrics: item.metrics
+                metrics: item.metrics,
+                logs: item.logs
               }))
             }
           ]);
         }
         if (event.type === "RunCompleted" || event.type === "RunFailed") {
           unsubscribe();
-          resolve4(event);
+          resolve5(event);
         }
       });
     });
@@ -1289,30 +1441,42 @@ function RunView({
           "ms)"
         ] })
       ] }),
-      tc.evaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, children: /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
-        item.evaluatorName,
-        ":",
-        " ",
-        /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
-        " ",
-        item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
-          formatScorePart(s),
-          " "
-        ] }, s.id)),
-        item.metrics?.map((m) => {
-          const def = getMetricById(m.id);
-          if (!def)
-            return null;
-          const formatted = def.format(m.data);
-          return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
-            "[",
-            def.name ? `${def.name}: ` : "",
-            formatted,
-            "]",
+      tc.evaluatorScores.map((item) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", marginLeft: 2, children: [
+        /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { children: [
+          item.evaluatorName,
+          ":",
+          " ",
+          /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: item.passed ? "green" : "red", bold: true, children: item.passed ? "PASS" : "FAIL" }),
+          " ",
+          item.scores.map((s) => /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: scoreColor(toNumericScore(s.data) ?? 0), children: [
+            formatScorePart(s),
             " "
-          ] }, m.id);
-        })
-      ] }) }, item.evaluatorId))
+          ] }, s.id)),
+          item.metrics?.map((m) => {
+            const def = getMetricById(m.id);
+            if (!def)
+              return null;
+            const formatted = def.format(m.data);
+            return /* @__PURE__ */ jsxRuntime.jsxs(ink.Text, { color: "gray", children: [
+              "[",
+              def.name ? `${def.name}: ` : "",
+              formatted,
+              "]",
+              " "
+            ] }, m.id);
+          })
+        ] }),
+        !item.passed && item.logs && item.logs.length > 0 && /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { marginLeft: 2, flexDirection: "column", children: item.logs.map(
+          (log, logIdx) => log.type === "diff" ? /* @__PURE__ */ jsxRuntime.jsx(ink.Box, { flexDirection: "column", children: getDiffLines(log).map(({ type, line }, lineIdx) => /* @__PURE__ */ jsxRuntime.jsx(
+            ink.Text,
+            {
+              color: type === "remove" ? "red" : type === "add" ? "green" : "gray",
+              children: line
+            },
+            lineIdx
+          )) }, logIdx) : null
+        ) })
+      ] }, item.evaluatorId))
     ] }, i)) }),
     phase === "completed" && summary && /* @__PURE__ */ jsxRuntime.jsxs(ink.Box, { flexDirection: "column", children: [
       /* @__PURE__ */ jsxRuntime.jsx(ink.Text, { color: "cyan", bold: true, children: "Run Summary" }),
@@ -1541,7 +1705,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     );
   }
   let spinnerTimer;
-  const done = new Promise((resolve4) => {
+  const done = new Promise((resolve5) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
@@ -1561,6 +1725,17 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
               item.metrics
             )
           );
+          if (!item.passed && item.logs && item.logs.length > 0) {
+            for (const log of item.logs) {
+              if (log.type === "diff") {
+                const useColor = process.stdout.isTTY;
+                for (const { type, line } of getDiffLines(log)) {
+                  const colored = useColor && type === "remove" ? colorize(`      ${line}`, ansi2.red) : useColor && type === "add" ? colorize(`      ${line}`, ansi2.green) : `      ${line}`;
+                  console.log(colored);
+                }
+              }
+            }
+          }
           const numeric = toNumericScoreFromScores(item.scores);
           if (numeric !== void 0) {
             const current = aggregates.get(item.evaluatorId) ?? {
@@ -1591,7 +1766,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         runFinished = true;
         clearLine();
         unsubscribe();
-        resolve4(event);
+        resolve5(event);
       }
     });
   });
@@ -1669,7 +1844,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi2.dim)}`);
 }
 async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(RunView, {
         runner,
@@ -1680,7 +1855,7 @@ async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })