@m4trix/evals 0.15.0 → 0.17.0

package/dist/index.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
 var effect = require('effect');
-var diff = require('diff');
+var jsonDiff = require('json-diff');
 var crypto = require('crypto');
 var fs = require('fs');
 var path = require('path');
@@ -660,46 +660,48 @@ var binaryScore = Score.of({
   },
   aggregate: aggregateAll
 });
-function toJsonLines(value) {
+function createDiffString(expected, actual, diffOptions) {
+  const opts = { ...diffOptions, color: false };
+  const result = jsonDiff.diffString(expected, actual, opts);
+  return typeof result === "string" ? result : "";
+}
+function formatLogMessage(msg) {
+  if (typeof msg === "string")
+    return msg;
   try {
-    return JSON.stringify(value, null, 2);
+    if (msg !== null && typeof msg === "object") {
+      return JSON.stringify(msg, null, 2);
+    }
+    return String(msg);
   } catch {
-    return String(value);
+    return String(msg);
   }
 }
-function formatDiffString(changes) {
-  const lines = [];
-  for (const part of changes) {
-    const prefix = part.added ? "+" : part.removed ? "-" : " ";
-    const partLines = part.value.split("\n");
-    if (partLines[partLines.length - 1] === "") {
-      partLines.pop();
-    }
-    for (const line of partLines) {
-      lines.push(`${prefix} ${line}`);
-    }
-  }
-  return lines.join("\n");
+function createLogEntry(message, options) {
+  return {
+    type: "log",
+    label: options?.label,
+    message: formatLogMessage(message)
+  };
 }
-function createDiffString(expected, actual) {
-  const expectedStr = toJsonLines(expected);
-  const actualStr = toJsonLines(actual);
-  const changes = diff.diffLines(expectedStr, actualStr);
-  return formatDiffString(changes);
+function getLogLines(entry) {
+  return entry.message.split("\n");
 }
 function createDiffLogEntry(expected, actual, options) {
-  const diff = createDiffString(expected, actual);
+  const { label, ...diffOpts } = options ?? {};
+  const diff = createDiffString(expected, actual, diffOpts);
   return {
     type: "diff",
-    label: options?.label,
+    label,
     expected,
     actual,
     diff: diff || "(no differences)"
   };
 }
 function printJsonDiff(expected, actual, options = {}) {
-  const diff = createDiffString(expected, actual);
-  if (options.color) {
+  const { color = true, ...diffOpts } = options;
+  const diff = createDiffString(expected, actual, diffOpts);
+  if (color) {
     const lines = diff.split("\n").map((line) => {
       const trimmed = line.trimStart();
       if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -1055,6 +1057,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
+      const evaluatorRunId = `run-${crypto.randomUUID()}`;
       const started = Date.now();
       const evaluatorScores = [];
       let testCaseError;
@@ -1069,6 +1072,9 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
           const logDiff = (expected, actual, options) => {
             logs.push(createDiffLogEntry(expected, actual, options));
           };
+          const log = (message, options) => {
+            logs.push(createLogEntry(message, options));
+          };
           const ctx = yield* effect.Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
@@ -1078,7 +1084,13 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
-                logDiff
+                meta: {
+                  triggerId: task.triggerId,
+                  runId: evaluatorRunId,
+                  datasetId: task.datasetId
+                },
+                logDiff,
+                log
               })
             )
           );
@@ -1536,6 +1548,7 @@ var EffectRunner = class {
       (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
       0
     );
+    const triggerId = request.triggerId ?? `trg-${crypto.randomUUID()}`;
     const runId = `run-${crypto.randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1577,6 +1590,7 @@ var EffectRunner = class {
     await effect.Effect.runPromise(
       effect.Queue.offer(this.runQueue, {
         runId,
+        triggerId,
         datasetId: request.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
@@ -1660,9 +1674,11 @@ exports.Metric = Metric;
 exports.Score = Score;
 exports.TestCase = TestCase;
 exports.binaryScore = binaryScore;
+exports.createLogEntry = createLogEntry;
 exports.createRunner = createRunner;
 exports.defaultRunnerConfig = defaultRunnerConfig;
 exports.defineConfig = defineConfig;
+exports.getLogLines = getLogLines;
 exports.getMetricById = getMetricById;
 exports.getScoreById = getScoreById;
 exports.latencyMetric = latencyMetric;