@m4trix/evals 0.11.0 → 0.12.0

package/dist/cli-simple.cjs

@@ -586,6 +586,120 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     artifactPath: task.snapshot.artifactPath
   });
 });
+async function loadRunSnapshotsFromArtifacts(config) {
+  const baseDir = path.resolve(config.artifactDirectory);
+  let entries;
+  try {
+    entries = await promises.readdir(baseDir);
+  } catch {
+    return [];
+  }
+  const jsonlFiles = entries.filter((name) => name.endsWith(".jsonl"));
+  const snapshots = [];
+  for (const fileName of jsonlFiles) {
+    const filePath = path.join(baseDir, fileName);
+    try {
+      const snapshot = await parseArtifactToSnapshot(filePath, config);
+      if (snapshot) {
+        snapshots.push(snapshot);
+      }
+    } catch {
+    }
+  }
+  return snapshots.sort((a, b) => b.queuedAt - a.queuedAt);
+}
+async function parseArtifactToSnapshot(filePath, _config) {
+  const content = await promises.readFile(filePath, "utf8");
+  const lines = content.split("\n").filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+  let runQueued = null;
+  let runCompleted = null;
+  let runFailed = null;
+  let runStarted = null;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const type = event.type;
+      if (type === "RunQueued") {
+        runQueued = {
+          runId: event.runId,
+          datasetId: event.datasetId,
+          datasetName: event.datasetName,
+          evaluatorIds: event.evaluatorIds,
+          totalTestCases: event.totalTestCases ?? 0,
+          artifactPath: event.artifactPath ?? filePath,
+          ts: event.ts
+        };
+      }
+      if (type === "RunStarted") {
+        runStarted = { startedAt: event.startedAt };
+      }
+      if (type === "RunCompleted") {
+        runCompleted = {
+          passedTestCases: event.passedTestCases,
+          failedTestCases: event.failedTestCases,
+          totalTestCases: event.totalTestCases,
+          finishedAt: event.finishedAt
+        };
+      }
+      if (type === "RunFailed") {
+        runFailed = {
+          finishedAt: event.finishedAt,
+          errorMessage: event.errorMessage
+        };
+      }
+    } catch {
+    }
+  }
+  if (!runQueued) {
+    return null;
+  }
+  const artifactPath = filePath;
+  const status = runFailed ? "failed" : runCompleted ? "completed" : runStarted ? "running" : "queued";
+  const progress = aggregateTestCaseProgress(lines);
+  const completedTestCases = runCompleted?.totalTestCases ?? progress.completedTestCases;
+  const passedTestCases = runCompleted?.passedTestCases ?? progress.passedTestCases;
+  const failedTestCases = runCompleted?.failedTestCases ?? progress.failedTestCases;
+  return {
+    runId: runQueued.runId,
+    datasetId: runQueued.datasetId,
+    datasetName: runQueued.datasetName,
+    evaluatorIds: runQueued.evaluatorIds,
+    queuedAt: runQueued.ts ?? 0,
+    startedAt: runStarted?.startedAt,
+    finishedAt: runCompleted?.finishedAt ?? runFailed?.finishedAt,
+    totalTestCases: runQueued.totalTestCases,
+    completedTestCases,
+    passedTestCases,
+    failedTestCases,
+    status,
+    artifactPath,
+    errorMessage: runFailed?.errorMessage
+  };
+}
+function aggregateTestCaseProgress(lines) {
+  let completedTestCases = 0;
+  let passedTestCases = 0;
+  let failedTestCases = 0;
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.type === "TestCaseProgress") {
+        const ev = event;
+        completedTestCases = ev.completedTestCases ?? completedTestCases;
+        if (ev.passed) {
+          passedTestCases += 1;
+        } else {
+          failedTestCases += 1;
+        }
+      }
+    } catch {
+    }
+  }
+  return { completedTestCases, passedTestCases, failedTestCases };
+}
 async function appendJsonLine(artifactPath, payload) {
   await promises.mkdir(path.dirname(artifactPath), { recursive: true });
   await promises.appendFile(artifactPath, `${JSON.stringify(payload)}
@@ -842,6 +956,9 @@ var EffectRunner = class {
       (a, b) => b.queuedAt - a.queuedAt
     );
   }
+  async loadRunSnapshotsFromArtifacts() {
+    return loadRunSnapshotsFromArtifacts(this.config);
+  }
   async shutdown() {
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.schedulerFiber));
     await effect.Effect.runPromise(effect.Fiber.interrupt(this.persistenceFiber));
@@ -973,7 +1090,7 @@ function GenerateView({
         return;
       }
       const { writeFile: writeFile2 } = await import('fs/promises');
-      const { join: join3, parse: parse2, resolve: resolve4 } = await import('path');
+      const { join: join4, parse: parse2, resolve: resolve5 } = await import('path');
       const testCases = await runner.collectDatasetTestCases(dataset.id);
       const payload = testCases.map((item) => {
         const tc = item.testCase;
@@ -983,9 +1100,9 @@ function GenerateView({
           output: typeof tc.getOutput === "function" ? tc.getOutput() : void 0
         };
       });
-      const absoluteDatasetPath = resolve4(process.cwd(), dataset.filePath);
+      const absoluteDatasetPath = resolve5(process.cwd(), dataset.filePath);
       const parsed = parse2(absoluteDatasetPath);
-      const outputPath = join3(parsed.dir, `${parsed.name}.cases.json`);
+      const outputPath = join4(parsed.dir, `${parsed.name}.cases.json`);
       await writeFile2(
         outputPath,
         `${JSON.stringify(payload, null, 2)}
@@ -1060,7 +1177,7 @@ async function generateDatasetJsonCommandPlain(runner, datasetName) {
   console.log(`Wrote ${outputPath}`);
 }
 async function generateDatasetJsonCommandInk(runner, datasetName) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(GenerateView, {
         runner,
@@ -1070,7 +1187,7 @@ async function generateDatasetJsonCommandInk(runner, datasetName) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })
@@ -1198,7 +1315,7 @@ function RunView({
     const aggregates = /* @__PURE__ */ new Map();
     let overallScoreTotal = 0;
     let overallScoreCount = 0;
-    const done = new Promise((resolve4) => {
+    const done = new Promise((resolve5) => {
       const unsubscribe = runner.subscribeRunEvents((event) => {
         if (event.type === "TestCaseProgress") {
           const numericScores = event.evaluatorScores.map((item) => toNumericScoreFromScores(item.scores)).filter((item) => item !== void 0);
@@ -1244,7 +1361,7 @@ function RunView({
         }
         if (event.type === "RunCompleted" || event.type === "RunFailed") {
           unsubscribe();
-          resolve4(event);
+          resolve5(event);
         }
       });
     });
@@ -1588,7 +1705,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
     );
   }
   let spinnerTimer;
-  const done = new Promise((resolve4) => {
+  const done = new Promise((resolve5) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
@@ -1649,7 +1766,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
         runFinished = true;
         clearLine();
         unsubscribe();
-        resolve4(event);
+        resolve5(event);
       }
     });
   });
@@ -1727,7 +1844,7 @@ async function runSimpleEvalCommandPlain(runner, datasetName, evaluatorPattern)
   console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi2.dim)}`);
 }
 async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const app = ink.render(
       React2__default.default.createElement(RunView, {
         runner,
@@ -1738,7 +1855,7 @@ async function runSimpleEvalCommandInk(runner, datasetName, evaluatorPattern) {
           if (err) {
             reject(err);
           } else {
-            resolve4();
+            resolve5();
           }
         }
       })