@m4trix/evals 0.20.0 → 0.21.0

package/dist/index.cjs

@@ -1128,13 +1128,27 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
   return effect.Effect.gen(function* () {
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
       const evaluatorRunId = `run-${crypto.randomUUID()}`;
       const started = Date.now();
+      const startedEvaluations = yield* effect.Ref.modify(startedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      yield* publishEvent({
+        type: "TestCaseStarted",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        startedTestCases: startedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns
+      });
       const evaluatorScores = [];
       let testCaseError;
       const output = readOutput(testCaseItem.testCase);
@@ -1280,6 +1294,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
   );
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* effect.Ref.make(0);
+  const startedRef = yield* effect.Ref.make(0);
   const passedRef = yield* effect.Ref.make(0);
   const failedRef = yield* effect.Ref.make(0);
   const processTestCase = (testCaseItem) => processOneTestCase(
@@ -1289,6 +1304,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     publishEvent,
     persistenceQueue,
     updateSnapshot,
+    startedRef,
     completedRef,
     passedRef,
     failedRef