@m4trix/evals 0.15.0 → 0.17.0

package/dist/index.d.ts

@@ -167,6 +167,30 @@ declare class Dataset {
     matchesTestCase(testCase: TestCase<unknown>, filePath: string): boolean;
 }
 
+/**
+ * Options for customizing JSON diff output. Passed to logDiff, createDiffLogEntry, and printJsonDiff.
+ * @see https://www.npmjs.com/package/json-diff
+ */
+interface JsonDiffOptions {
+    /** Include equal sections of the document, not just deltas */
+    full?: boolean;
+    /** Sort primitive values in arrays before comparing */
+    sort?: boolean;
+    /** Compare only keys, ignore value differences */
+    keysOnly?: boolean;
+    /** Always output these keys when their parent object has any diff (comma-separated or array) */
+    outputKeys?: string | string[];
+    /** Output only new/updated values (no - lines) */
+    outputNewOnly?: boolean;
+    /** Exclude these keys from comparison (comma-separated or array) */
+    excludeKeys?: string | string[];
+    /** Include unchanged values in output */
+    keepUnchangedValues?: boolean;
+    /** Round floats to this many decimals before comparing */
+    precision?: number;
+    /** Max ... elisions in a row before collapsing */
+    maxElisions?: number;
+}
 interface DiffLogEntry {
     type: 'diff';
     label?: string;
@@ -174,7 +198,26 @@ interface DiffLogEntry {
     actual: unknown;
     diff: string;
 }
-interface PrintJsonDiffOptions {
+interface LogEntry {
+    type: 'log';
+    label?: string;
+    message: string;
+}
+type EvaluatorLogEntry = DiffLogEntry | LogEntry;
+/**
+ * Creates a LogEntry for storage in run artifacts. Use for logging objects or text.
+ */
+declare function createLogEntry(message: unknown, options?: {
+    label?: string;
+}): LogEntry;
+/**
+ * Returns lines from a log entry for display.
+ */
+declare function getLogLines(entry: LogEntry): string[];
+interface CreateDiffLogEntryOptions extends JsonDiffOptions {
+    label?: string;
+}
+interface PrintJsonDiffOptions extends JsonDiffOptions {
     /** Enable ANSI colors (default: true) */
     color?: boolean;
 }
@@ -191,12 +234,27 @@ interface EvalMiddleware<TCtx> {
     name: string;
     resolve: () => TCtx | Promise<TCtx>;
 }
+interface EvaluateMeta {
+    /** Identifier of the trigger that started the run (for example, a CLI invocation). */
+    triggerId: string;
+    /**
+     * Identifier of the current test-case execution shared across all evaluators
+     * for this specific test-case run.
+     */
+    runId: string;
+    /** Identifier of the dataset currently being evaluated. */
+    datasetId: string;
+}
 interface EvaluateArgs<TInput, TOutput = unknown, TCtx = Record<string, never>> {
     input: TInput;
     ctx: TCtx;
     output?: TOutput;
+    /** Metadata about the current evaluator invocation. */
+    meta: EvaluateMeta;
     /** Records a diff for this test case; stored in run artifact and shown by CLI */
-    logDiff: (expected: unknown, actual: unknown, options?: {
+    logDiff: (expected: unknown, actual: unknown, options?: CreateDiffLogEntryOptions) => void;
+    /** Logs a message or object for this test case; stored in run artifact and shown by CLI */
+    log: (message: unknown, options?: {
         label?: string;
     }) => void;
 }
@@ -304,6 +362,11 @@ interface SearchTestCasesQuery {
     excludedPaths?: ReadonlyArray<string | RegExp>;
 }
 interface RunDatasetRequest {
+    /**
+     * Identifier for what triggered the run request (for example, a CLI command).
+     * When omitted, the runner generates one in the format `trg-[uuid]`.
+     */
+    triggerId?: string;
     datasetId: string;
     evaluatorIds: ReadonlyArray<string>;
     concurrency?: number;
@@ -352,7 +415,7 @@ type RunnerEvent = {
         scores: ReadonlyArray<ScoreItem>;
         passed: boolean;
         metrics?: ReadonlyArray<MetricItem>;
-        logs?: ReadonlyArray<DiffLogEntry>;
+        logs?: ReadonlyArray<EvaluatorLogEntry>;
     }>;
     output?: unknown;
     errorMessage?: string;
@@ -424,4 +487,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, CreateDiffLogEntryOptions, Dataset, DiffLogEntry, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, EvaluateMeta, Evaluator, EvaluatorLogEntry, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, JsonDiffOptions, LatencyData, LogEntry, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Effect, PubSub, Queue, Fiber, Ref } from 'effect';
 export { Schema as S } from 'effect';
-import { diffLines } from 'diff';
+import { diffString } from 'json-diff';
 import { randomUUID } from 'crypto';
 import { existsSync } from 'fs';
 import { resolve as resolve$1, relative, join, dirname } from 'path';
@@ -638,46 +638,48 @@ var binaryScore = Score.of({
   },
   aggregate: aggregateAll
 });
-function toJsonLines(value) {
+function createDiffString(expected, actual, diffOptions) {
+  const opts = { ...diffOptions, color: false };
+  const result = diffString(expected, actual, opts);
+  return typeof result === "string" ? result : "";
+}
+function formatLogMessage(msg) {
+  if (typeof msg === "string")
+    return msg;
   try {
-    return JSON.stringify(value, null, 2);
+    if (msg !== null && typeof msg === "object") {
+      return JSON.stringify(msg, null, 2);
+    }
+    return String(msg);
   } catch {
-    return String(value);
+    return String(msg);
   }
 }
-function formatDiffString(changes) {
-  const lines = [];
-  for (const part of changes) {
-    const prefix = part.added ? "+" : part.removed ? "-" : " ";
-    const partLines = part.value.split("\n");
-    if (partLines[partLines.length - 1] === "") {
-      partLines.pop();
-    }
-    for (const line of partLines) {
-      lines.push(`${prefix} ${line}`);
-    }
-  }
-  return lines.join("\n");
+function createLogEntry(message, options) {
+  return {
+    type: "log",
+    label: options?.label,
+    message: formatLogMessage(message)
+  };
 }
-function createDiffString(expected, actual) {
-  const expectedStr = toJsonLines(expected);
-  const actualStr = toJsonLines(actual);
-  const changes = diffLines(expectedStr, actualStr);
-  return formatDiffString(changes);
+function getLogLines(entry) {
+  return entry.message.split("\n");
 }
 function createDiffLogEntry(expected, actual, options) {
-  const diff = createDiffString(expected, actual);
+  const { label, ...diffOpts } = options ?? {};
+  const diff = createDiffString(expected, actual, diffOpts);
   return {
     type: "diff",
-    label: options?.label,
+    label,
     expected,
     actual,
     diff: diff || "(no differences)"
   };
 }
 function printJsonDiff(expected, actual, options = {}) {
-  const diff = createDiffString(expected, actual);
-  if (options.color) {
+  const { color = true, ...diffOpts } = options;
+  const diff = createDiffString(expected, actual, diffOpts);
+  if (color) {
     const lines = diff.split("\n").map((line) => {
       const trimmed = line.trimStart();
       if (trimmed.startsWith("-") && !trimmed.startsWith("---")) {
@@ -1033,6 +1035,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
     const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
     const rerunPassed = [];
     for (let r = 0; r < reruns; r++) {
+      const evaluatorRunId = `run-${randomUUID()}`;
       const started = Date.now();
       const evaluatorScores = [];
       let testCaseError;
@@ -1047,6 +1050,9 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
           const logDiff = (expected, actual, options) => {
             logs.push(createDiffLogEntry(expected, actual, options));
           };
+          const log = (message, options) => {
+            logs.push(createLogEntry(message, options));
+          };
           const ctx = yield* Effect.promise(
             () => Promise.resolve(evaluator.resolveContext())
           );
@@ -1056,7 +1062,13 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
                 input: testCaseItem.testCase.getInput(),
                 ctx,
                 output,
-                logDiff
+                meta: {
+                  triggerId: task.triggerId,
+                  runId: evaluatorRunId,
+                  datasetId: task.datasetId
+                },
+                logDiff,
+                log
               })
             )
           );
@@ -1514,6 +1526,7 @@ var EffectRunner = class {
       (sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
       0
     );
+    const triggerId = request.triggerId ?? `trg-${randomUUID()}`;
     const runId = `run-${randomUUID()}`;
     const artifactPath = createArtifactPath(
       this.config.artifactDirectory,
@@ -1555,6 +1568,7 @@ var EffectRunner = class {
     await Effect.runPromise(
       Queue.offer(this.runQueue, {
         runId,
+        triggerId,
         datasetId: request.datasetId,
         dataset: dataset.dataset,
         evaluators: selectedEvaluators,
@@ -1628,6 +1642,6 @@ var EffectRunner = class {
   }
 };
 
-export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { Dataset, Evaluator, Metric, Score, TestCase, binaryScore, createLogEntry, createRunner, defaultRunnerConfig, defineConfig, getLogLines, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
 //# sourceMappingURL=out.js.map
 //# sourceMappingURL=index.js.map