@m4trix/evals 0.20.0 → 0.21.1

package/dist/index.d.ts

@@ -169,10 +169,9 @@ declare class Dataset {
 
 /**
  * Options for customizing JSON diff output. Passed to logDiff, createDiffLogEntry, and printJsonDiff.
- * @see https://www.npmjs.com/package/json-diff
  */
 interface JsonDiffOptions {
-    /** Include equal sections of the document, not just deltas */
+    /** Include equal sections of the document, not just deltas (always true with current implementation) */
     full?: boolean;
     /** Sort primitive values in arrays before comparing */
     sort?: boolean;
@@ -437,6 +436,15 @@ type RunnerEvent = {
     type: 'RunStarted';
     runId: string;
     startedAt: number;
+} | {
+    type: 'TestCaseStarted';
+    runId: string;
+    testCaseId: string;
+    testCaseName: string;
+    startedTestCases: number;
+    totalTestCases: number;
+    rerunIndex: number;
+    rerunTotal: number;
 } | {
     type: 'TestCaseProgress';
     runId: string;

package/dist/index.js

@@ -1,6 +1,7 @@
 import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 export { Schema as S } from 'effect';
-import { diffString } from 'json-diff';
+import { diffLines } from 'diff';
+import stringify from 'fast-json-stable-stringify';
 import { randomUUID } from 'crypto';
 import { existsSync } from 'fs';
 import { resolve as resolve$1, relative, join, dirname } from 'path';
@@ -708,10 +709,102 @@ var binaryScore = Score.of({
   },
   aggregateValues: Score.aggregate.all
 });
+function preprocessForDiff(value, options) {
+  if (options?.sort && Array.isArray(value)) {
+    return [...value].sort((a, b) => {
+      const aStr = stringify(preprocessForDiff(a, options));
+      const bStr = stringify(preprocessForDiff(b, options));
+      return aStr.localeCompare(bStr);
+    }).map((item) => preprocessForDiff(item, options));
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && options?.excludeKeys) {
+    const keys = Array.isArray(options.excludeKeys) ? options.excludeKeys : options.excludeKeys.split(",").map((k) => k.trim());
+    const filtered = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (!keys.includes(k)) {
+        filtered[k] = preprocessForDiff(v, options);
+      }
+    }
+    return filtered;
+  }
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = preprocessForDiff(v, options);
+    }
+    return result;
+  }
+  if (typeof value === "number" && options?.precision !== void 0) {
+    return Number(value.toFixed(options.precision));
+  }
+  return value;
+}
+function toPrettyJson(value) {
+  const str = stringify(value);
+  try {
+    const parsed = JSON.parse(str);
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return str;
+  }
+}
+function formatDiffParts(parts) {
+  const lines = [];
+  for (const part of parts) {
+    const prefix = part.added ? "+ " : part.removed ? "- " : "";
+    const partLines = part.value.split("\n");
+    for (let i = 0; i < partLines.length; i++) {
+      const line = partLines[i];
+      if (i === partLines.length - 1 && line === "")
+        continue;
+      lines.push(prefix + line);
+    }
+  }
+  return lines.join("\n");
+}
 function createDiffString(expected, actual, diffOptions) {
-  const opts = { ...diffOptions, color: false };
-  const result = diffString(expected, actual, opts);
-  return typeof result === "string" ? result : "";
+  const expectedProcessed = preprocessForDiff(expected, diffOptions);
+  const actualProcessed = preprocessForDiff(actual, diffOptions);
+  if (diffOptions?.keysOnly) {
+    const expectedKeys = JSON.stringify(
+      extractKeys(expectedProcessed),
+      null,
+      2
+    );
+    const actualKeys = JSON.stringify(
+      extractKeys(actualProcessed),
+      null,
+      2
+    );
+    const parts2 = diffLines(expectedKeys, actualKeys);
+    return formatDiffParts(parts2);
+  }
+  const expectedStr = toPrettyJson(expectedProcessed);
+  const actualStr = toPrettyJson(actualProcessed);
+  if (expectedStr === actualStr) {
+    return "";
+  }
+  const parts = diffLines(expectedStr, actualStr);
+  if (diffOptions?.outputNewOnly) {
+    const filtered = parts.filter(
+      (p) => p.added === true
+    );
+    return formatDiffParts(filtered);
+  }
+  return formatDiffParts(parts);
+}
+function extractKeys(value) {
+  if (value === null || typeof value !== "object") {
+    return "\xB7";
+  }
+  if (Array.isArray(value)) {
+    return value.map(extractKeys);
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = extractKeys(v);
+  }
+  return result;
 }
 function formatLogMessage(msg) {
   if (typeof msg === "string")
@@ -1106,13 +1199,27 @@ function createArtifactPath(artifactDirectory, datasetId, runId) {
     `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`
   );
 }
-function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, completedRef, passedRef, failedRef) {
+function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent, persistenceQueue, updateSnapshot, startedRef, completedRef, passedRef, failedRef) {
   return Effect.gen(function* () {
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
       const evaluatorRunId = `run-${randomUUID()}`;
       const started = Date.now();
+      const startedEvaluations = yield* Ref.modify(startedRef, (n) => [
+        n + 1,
+        n + 1
+      ]);
+      yield* publishEvent({
+        type: "TestCaseStarted",
+        runId: task.runId,
+        testCaseId: testCaseItem.id,
+        testCaseName: testCaseItem.testCase.getName(),
+        startedTestCases: startedEvaluations,
+        totalTestCases: totalEvaluations,
+        rerunIndex: r + 1,
+        rerunTotal: reruns
+      });
       const evaluatorScores = [];
       let testCaseError;
       const output = readOutput(testCaseItem.testCase);
@@ -1258,6 +1365,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
   );
   const maxConcurrency = Math.max(1, task.maxConcurrency ?? 1);
   const completedRef = yield* Ref.make(0);
+  const startedRef = yield* Ref.make(0);
   const passedRef = yield* Ref.make(0);
   const failedRef = yield* Ref.make(0);
   const processTestCase = (testCaseItem) => processOneTestCase(
@@ -1267,6 +1375,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     publishEvent,
     persistenceQueue,
     updateSnapshot,
+    startedRef,
     completedRef,
     passedRef,
     failedRef