@anvia/core 1.0.7 → 1.0.9

package/dist/evals/index.js

@@ -10,11 +10,11 @@ import {
 } from "../chunk-IZNOP6JG.js";
 import {
   extract
-} from "../chunk-DHJL44R2.js";
-import "../chunk-D4PACGPA.js";
+} from "../chunk-JTJU56ZV.js";
+import "../chunk-4UQALF5Y.js";
 import {
   Usage
-} from "../chunk-TB5EKZM7.js";
+} from "../chunk-OI3LSMJG.js";
 import "../chunk-7JLAIN6E.js";
 import "../chunk-QGX73TSQ.js";
 import {
@@ -32,18 +32,15 @@ function defaultOutputValue(output) {
   }
   return output;
 }
-function stableComparable(value) {
-  if (typeof value === "string") {
-    return value;
-  }
-  return JSON.stringify(value);
+function evalValuesEqual(left, right) {
+  return deepEqual(left, right, /* @__PURE__ */ new WeakMap());
 }
 function formatValue(value) {
   if (typeof value === "string") {
     return value;
   }
   try {
-    return JSON.stringify(value);
+    return JSON.stringify(value) ?? String(value);
   } catch {
     return String(value);
   }
@@ -51,6 +48,34 @@ function formatValue(value) {
 function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function deepEqual(left, right, seen) {
+  if (Object.is(left, right)) return true;
+  if (typeof left !== "object" || left === null || typeof right !== "object" || right === null) {
+    return false;
+  }
+  if (Object.getPrototypeOf(left) !== Object.getPrototypeOf(right)) return false;
+  if (left instanceof Date && right instanceof Date) return left.getTime() === right.getTime();
+  if (left instanceof RegExp && right instanceof RegExp) return String(left) === String(right);
+  if (!Array.isArray(left) && Object.getPrototypeOf(left) !== Object.prototype && Object.getPrototypeOf(left) !== null) {
+    return false;
+  }
+  if (seen.get(left) === right) return true;
+  seen.set(left, right);
+  const leftKeys = Reflect.ownKeys(left);
+  const rightKeys = Reflect.ownKeys(right);
+  if (leftKeys.length !== rightKeys.length) return false;
+  for (const key of leftKeys) {
+    if (!Object.hasOwn(right, key)) return false;
+    if (!deepEqual(
+      left[key],
+      right[key],
+      seen
+    )) {
+      return false;
+    }
+  }
+  return true;
+}
 
 // src/evals/judge.ts
 async function runJudge(args) {
@@ -134,16 +159,16 @@ var EvalOutcome = {
     if (options.score !== void 0) {
       outcome.score = options.score;
     }
-    if (options.comment !== void 0) {
-      outcome.comment = options.comment;
-    }
-    if (options.metadata !== void 0) {
-      outcome.metadata = options.metadata;
-    }
-    if (options.usage !== void 0) {
-      outcome.usage = options.usage;
-    }
+    if (options.comment !== void 0) outcome.comment = options.comment;
+    if (options.metadata !== void 0) outcome.metadata = options.metadata;
+    if (options.usage !== void 0) outcome.usage = options.usage;
+    if (options.kind !== void 0) outcome.kind = options.kind;
+    if (options.error !== void 0) outcome.error = options.error;
     return outcome;
+  },
+  fromError(error, kind = "metric") {
+    const reason = error instanceof Error ? error.message : String(error);
+    return EvalOutcome.invalid(reason, { kind, error });
   }
 };
 
@@ -154,12 +179,22 @@ function selectPromptOutput(args) {
   }
   return args.output.output;
 }
+function selectEvalCaseIds(result, outcomes = ["fail", "invalid"]) {
+  const selected = new Set(outcomes);
+  return result.results.filter((caseResult) => selected.has(caseResult.outcome)).map((caseResult) => caseResult.case.id);
+}
 async function resolveActual(selector, args) {
   return selector === void 0 ? defaultOutputValue(args.output) : selector(args);
 }
 async function resolveActualText(selector, args) {
   const value = selector === void 0 ? defaultOutputValue(args.output) : await selector(args);
-  return typeof value === "string" ? value : JSON.stringify(value);
+  if (typeof value === "string") return value;
+  try {
+    const serialized = JSON.stringify(value);
+    if (serialized !== void 0) return serialized;
+  } catch {
+  }
+  throw new TypeError("Text metric actual value must be a string or JSON-serializable value.");
 }
 async function resolveExpected(selectorOrValue, args) {
   if (selectorOrValue === void 0) {
@@ -260,7 +295,7 @@ ${actual}`,
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -301,7 +336,7 @@ function promptAlignment(options) {
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -363,7 +398,7 @@ function jsonCorrectness(options) {
           usage
         });
       } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
+        return EvalOutcome.fromError(error);
       }
     }
   );
@@ -402,7 +437,7 @@ function hallucination(options) {
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -478,7 +513,7 @@ ${actual}`,
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -527,7 +562,7 @@ function abstention(options) {
         };
         return category === "correct_abstention" || category === "correct_grounded_answer" ? EvalOutcome.pass(category, outcomeOptions) : EvalOutcome.fail(category, outcomeOptions);
       } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
+        return EvalOutcome.fromError(error);
       }
     }
   };
@@ -677,7 +712,7 @@ ${actual}`,
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -761,7 +796,7 @@ function gEval(options) {
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -810,7 +845,7 @@ function turnRelevancy(options) {
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -882,7 +917,7 @@ function knowledgeRetention(options) {
         usage
       });
     } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
+      return EvalOutcome.fromError(error);
     }
   });
 }
@@ -1204,6 +1239,72 @@ function agentRunSettings(request) {
   return settings;
 }
 
+// src/evals/execution.ts
+var EvalTimeoutError = class extends Error {
+  constructor(timeoutMs) {
+    super(`Evaluation case exceeded its timeout of ${timeoutMs}ms.`);
+    this.timeoutMs = timeoutMs;
+    this.name = "EvalTimeoutError";
+  }
+  timeoutMs;
+};
+var EvalAbortError = class extends Error {
+  constructor(message = "Evaluation run was aborted.") {
+    super(message);
+    this.name = "EvalAbortError";
+  }
+};
+function createEvalCaseSignal(parent, timeoutMs) {
+  const controller = new AbortController();
+  const abortFromParent = () => controller.abort(parent?.reason ?? new EvalAbortError());
+  if (parent?.aborted === true) abortFromParent();
+  else parent?.addEventListener("abort", abortFromParent, { once: true });
+  const timeout = timeoutMs === void 0 ? void 0 : setTimeout(() => controller.abort(new EvalTimeoutError(timeoutMs)), timeoutMs);
+  return {
+    signal: controller.signal,
+    dispose() {
+      if (timeout !== void 0) clearTimeout(timeout);
+      parent?.removeEventListener("abort", abortFromParent);
+    }
+  };
+}
+function abortable(signal, operation) {
+  if (signal.aborted) return Promise.reject(abortReason(signal));
+  return new Promise((resolve, reject) => {
+    const onAbort = () => reject(abortReason(signal));
+    signal.addEventListener("abort", onAbort, { once: true });
+    operation.then(
+      (value) => {
+        signal.removeEventListener("abort", onAbort);
+        resolve(value);
+      },
+      (error) => {
+        signal.removeEventListener("abort", onAbort);
+        reject(error);
+      }
+    );
+  });
+}
+function createConcurrencyLimiter(concurrency) {
+  let active = 0;
+  const waiting = [];
+  return async (operation) => {
+    if (active >= concurrency) {
+      await new Promise((resolve) => waiting.push(resolve));
+    }
+    active += 1;
+    try {
+      return await operation();
+    } finally {
+      active -= 1;
+      waiting.shift()?.();
+    }
+  };
+}
+function abortReason(signal) {
+  return signal.reason ?? new EvalAbortError();
+}
+
 // src/evals/reporting.ts
 function projectEvalOutcome(outcome, dataType, projectScore) {
   const value = projectScoreValue(outcome, dataType, projectScore);
@@ -1296,15 +1397,30 @@ var EvalReporterDispatchError = class extends AggregateError {
     this.phase = phase;
   }
 };
+var EvalFailFastError = class extends Error {
+  constructor(caseId, outcome) {
+    super(`Evaluation stopped after case ${caseId} produced a required ${outcome} outcome.`);
+    this.caseId = caseId;
+    this.outcome = outcome;
+    this.name = "EvalFailFastError";
+  }
+  caseId;
+  outcome;
+};
 async function runEvalSuite(options) {
   validateSuiteOptions(options);
+  const selectedCases = selectCases(options);
+  const selectedOptions = {
+    ...options,
+    cases: selectedCases
+  };
   const startedAtMs = Date.now();
-  const run = resolveRun(options, startedAtMs);
+  const run = resolveRun(selectedOptions, startedAtMs);
   const reporters = options.reporters ?? [];
   const lifecycle = {
     run,
     suiteName: options.name,
-    caseCount: options.cases.length,
+    caseCount: selectedCases.length,
     metricNames: options.metrics.map((metric) => metric.name)
   };
   let reporterErrors;
@@ -1327,8 +1443,8 @@ async function runEvalSuite(options) {
   let results;
   let aggregates;
   try {
-    results = await runEvalCases(options, run);
-    aggregates = await aggregateResult(options, results);
+    results = await runEvalCases(selectedOptions, run);
+    aggregates = await aggregateResult(selectedOptions, results);
   } catch (error) {
     await notifyRunEnd(reporters, {
       ...lifecycle,
@@ -1350,6 +1466,11 @@ async function runEvalSuite(options) {
     durationMs: Date.now() - startedAtMs,
     reporterErrors
   };
+  result.reporterErrors.push(
+    ...results.flatMap(
+      (caseResult) => caseResult.metrics.flatMap((metricResult) => metricResult.reporterErrors)
+    )
+  );
   if (aggregates.cost !== void 0) {
     result.cost = aggregates.cost;
   }
@@ -1371,102 +1492,214 @@ async function runEvalSuite(options) {
   return result;
 }
 async function runEvalCases(options, run) {
-  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 1));
+  const targetConcurrency = options.targetConcurrency ?? options.concurrency ?? 1;
+  const metricConcurrency = options.metricConcurrency ?? options.concurrency ?? 1;
+  const workerConcurrency = Math.max(targetConcurrency, metricConcurrency);
+  const targetLimit = createConcurrencyLimiter(targetConcurrency);
+  const metricLimit = createConcurrencyLimiter(metricConcurrency);
   const results = Array(options.cases.length);
   let nextIndex = 0;
+  let completedCases = 0;
   let failure;
   async function worker() {
     while (failure === void 0 && nextIndex < options.cases.length) {
+      if (isAborted(options.signal)) {
+        failure ??= { error: suiteAbortReason(options.signal) };
+        break;
+      }
       const index = nextIndex;
       nextIndex += 1;
       try {
         results[index] = await runEvalCase(
           options,
           options.cases[index],
-          run
+          run,
+          targetLimit,
+          metricLimit,
+          () => completedCases
         );
+        completedCases += 1;
+        if (isAborted(options.signal)) {
+          failure ??= { error: suiteAbortReason(options.signal) };
+        }
+        if (options.failFast === true && results[index]?.outcome !== "pass") {
+          const result = results[index];
+          if (result !== void 0 && result.outcome !== "pass") {
+            failure ??= { error: new EvalFailFastError(result.case.id, result.outcome) };
+          }
+        }
       } catch (error) {
         failure ??= { error };
       }
     }
   }
   await Promise.all(
-    Array.from({ length: Math.min(concurrency, options.cases.length) }, () => worker())
+    Array.from({ length: Math.min(workerConcurrency, options.cases.length) }, () => worker())
   );
   if (failure !== void 0) throw failure.error;
   return results;
 }
-async function runEvalCase(options, testCase, run) {
-  let output;
-  let targetError;
+async function runEvalCase(options, testCase, run, targetLimit, metricLimit, completedCases) {
+  const caseStartedAt = performance.now();
+  const caseSignal = createEvalCaseSignal(options.signal, options.caseTimeoutMs);
   try {
-    output = await options.target(testCase.input, testCase);
-  } catch (error) {
-    targetError = error;
-  }
-  const traceResult = await resolveTrace(options, testCase, output, targetError);
-  const metrics = [];
-  for (const metric of options.metrics) {
-    const outcome = targetError === void 0 ? await safeEvaluate(options.name, testCase, output, metric) : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);
-    const reporterErrors = await reportOutcome({
-      run,
+    const progress = (event) => notifyProgress(options, event);
+    await progress({
+      type: "case-start",
+      suiteName: options.name,
+      case: testCase,
+      completedCases: completedCases(),
+      totalCases: options.cases.length
+    });
+    let output;
+    let targetError;
+    let targetStatus = "succeeded";
+    let targetDurationMs = 0;
+    try {
+      output = await targetLimit(async () => {
+        const targetStartedAt = performance.now();
+        try {
+          return await (caseSignal.signal.aborted ? Promise.reject(caseSignal.signal.reason) : abortable(
+            caseSignal.signal,
+            Promise.resolve(
+              options.target(testCase.input, testCase, { signal: caseSignal.signal })
+            )
+          ));
+        } finally {
+          targetDurationMs = performance.now() - targetStartedAt;
+        }
+      });
+    } catch (error) {
+      if (options.signal?.aborted === true) {
+        throw options.signal.reason ?? new EvalAbortError();
+      }
+      targetStatus = "failed";
+      targetError = error;
+    }
+    await progress({
+      type: "target-complete",
       suiteName: options.name,
-      testCase,
+      case: testCase,
+      targetStatus,
       output,
-      targetError,
-      metric,
-      outcome,
-      trace: traceResult.trace,
-      traceError: traceResult.error,
-      reporters: options.reporters ?? [],
-      reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
+      error: targetError,
+      durationMs: targetDurationMs,
+      completedCases: completedCases(),
+      totalCases: options.cases.length
     });
-    const metricResult = {
-      metricName: metric.name,
-      required: metric.required ?? true,
-      outcome,
-      reporterErrors
+    const traceResult = await resolveTrace(options, testCase, output, targetError, targetStatus);
+    const metrics = await Promise.all(
+      options.metrics.map(
+        (metric) => metricLimit(async () => {
+          const metricStartedAt = performance.now();
+          const outcome = targetStatus === "succeeded" ? await safeEvaluate(
+            options.name,
+            testCase,
+            output,
+            metric,
+            caseSignal.signal
+          ) : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`, {
+            kind: targetError instanceof EvalTimeoutError ? "timeout" : "target",
+            error: targetError
+          });
+          const durationMs = performance.now() - metricStartedAt;
+          const reporterErrors = await reportOutcome({
+            run,
+            suiteName: options.name,
+            testCase,
+            output,
+            targetError,
+            targetStatus,
+            metric,
+            outcome,
+            trace: traceResult.trace,
+            traceError: traceResult.error,
+            reporters: options.reporters ?? [],
+            reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
+          });
+          await progress({
+            type: "metric-complete",
+            suiteName: options.name,
+            case: testCase,
+            metricName: metric.name,
+            outcome,
+            durationMs,
+            completedCases: completedCases(),
+            totalCases: options.cases.length
+          });
+          const metricResult = {
+            metricName: metric.name,
+            required: metric.required ?? true,
+            outcome,
+            durationMs,
+            reporterErrors
+          };
+          if (metric.direction !== void 0) metricResult.direction = metric.direction;
+          if (metric.threshold !== void 0) metricResult.threshold = metric.threshold;
+          return metricResult;
+        })
+      )
+    );
+    const scores = Object.fromEntries(
+      metrics.map((metric) => [metric.metricName, metric.outcome])
+    );
+    const result = {
+      case: testCase,
+      outcome: caseOutcome(targetStatus, metrics),
+      targetStatus,
+      targetDurationMs,
+      durationMs: performance.now() - caseStartedAt,
+      metrics,
+      scores,
+      usage: emptyUsageSummary()
     };
-    if (metric.direction !== void 0) metricResult.direction = metric.direction;
-    if (metric.threshold !== void 0) metricResult.threshold = metric.threshold;
-    metrics.push(metricResult);
-  }
-  const scores = Object.fromEntries(
-    metrics.map((metric) => [metric.metricName, metric.outcome])
-  );
-  const result = {
-    case: testCase,
-    outcome: caseOutcome(targetError, metrics),
-    metrics,
-    scores
-  };
-  if (output !== void 0) {
-    result.output = output;
-  }
-  if (targetError !== void 0) {
-    result.targetError = targetError;
+    if (targetStatus === "succeeded") {
+      result.output = output;
+    }
+    if (targetStatus === "failed") {
+      result.targetError = targetError;
+    }
+    await progress({
+      type: "case-complete",
+      suiteName: options.name,
+      result,
+      completedCases: completedCases() + 1,
+      totalCases: options.cases.length
+    });
+    return result;
+  } finally {
+    caseSignal.dispose();
   }
-  return result;
 }
-async function resolveTrace(options, testCase, output, targetError) {
+async function resolveTrace(options, testCase, output, targetError, targetStatus) {
   try {
     const selector = options.trace ?? defaultEvalTraceSelector;
     const trace = await selector({
       suiteName: options.name,
       case: testCase,
       output,
-      targetError
+      targetError,
+      targetStatus
     });
     return trace === void 0 ? {} : { trace };
   } catch (error) {
     return { error };
   }
 }
-async function safeEvaluate(suiteName, testCase, output, metric) {
+async function safeEvaluate(suiteName, testCase, output, metric, signal) {
+  if (signal.aborted) {
+    return EvalOutcome.fromError(
+      signal.reason,
+      signal.reason instanceof EvalTimeoutError ? "timeout" : "metric"
+    );
+  }
   try {
-    return await metric.evaluate({ suiteName, case: testCase, output });
+    return await abortable(
+      signal,
+      Promise.resolve(metric.evaluate({ suiteName, case: testCase, output, signal }))
+    );
   } catch (error) {
-    return EvalOutcome.invalid(errorMessage(error));
+    return EvalOutcome.fromError(error, error instanceof EvalTimeoutError ? "timeout" : "metric");
   }
 }
 async function reportOutcome(args) {
@@ -1482,6 +1715,7 @@ async function reportOutcome(args) {
         case: args.testCase,
         output: args.output,
         targetError: args.targetError,
+        targetStatus: args.targetStatus,
         trace: args.trace,
         metric: args.metric,
         outcome: args.outcome
@@ -1567,13 +1801,20 @@ function countCaseOutcomes(results) {
 function emptyTotals() {
   return { total: 0, passed: 0, failed: 0, invalid: 0 };
 }
+function emptyUsageSummary() {
+  return {
+    target: Usage.empty(),
+    evaluation: Usage.empty(),
+    total: Usage.empty()
+  };
+}
 function statusKey(status) {
   if (status === "pass") return "passed";
   if (status === "fail") return "failed";
   return "invalid";
 }
-function caseOutcome(targetError, metrics) {
-  if (targetError !== void 0) return "invalid";
+function caseOutcome(targetStatus, metrics) {
+  if (targetStatus === "failed") return "invalid";
   const required = metrics.filter((metric) => metric.required);
   if (required.some((metric) => metric.outcome.outcome === "invalid")) return "invalid";
   if (required.some((metric) => metric.outcome.outcome === "fail")) return "fail";
@@ -1585,12 +1826,17 @@ async function aggregateResult(options, results) {
   let targetCost = 0;
   let evaluationCost = 0;
   for (const result of results) {
-    if (result.output !== void 0) {
+    let caseTargetUsage = Usage.empty();
+    let caseEvaluationUsage = Usage.empty();
+    let caseTargetCost = 0;
+    let caseEvaluationCost = 0;
+    if (result.targetStatus === "succeeded") {
       const usage2 = await resolveTargetUsage(options, result.case, result.output);
       if (usage2 !== void 0) {
+        caseTargetUsage = usage2;
         targetUsage = Usage.add(targetUsage, usage2);
         if (options.cost !== void 0) {
-          targetCost += await calculateCost(
+          caseTargetCost = await calculateCost(
             options.cost.calculate({
               kind: "target",
               suiteName: options.name,
@@ -1599,6 +1845,7 @@ async function aggregateResult(options, results) {
               usage: usage2
             })
           );
+          targetCost += caseTargetCost;
         }
       }
     }
@@ -1606,13 +1853,14 @@ async function aggregateResult(options, results) {
       const usage2 = metricResult.outcome.usage;
       if (usage2 === void 0) continue;
       assertUsage(usage2, `Evaluation usage for metric ${metricResult.metricName}`);
+      caseEvaluationUsage = Usage.add(caseEvaluationUsage, usage2);
       evaluationUsage = Usage.add(evaluationUsage, usage2);
-      if (options.cost !== void 0 && result.output !== void 0) {
+      if (options.cost !== void 0 && result.targetStatus === "succeeded") {
         const metric = options.metrics.find(
           (candidate) => candidate.name === metricResult.metricName
         );
         if (metric !== void 0) {
-          evaluationCost += await calculateCost(
+          const metricCost = await calculateCost(
             options.cost.calculate({
               kind: "evaluation",
               suiteName: options.name,
@@ -1622,9 +1870,25 @@ async function aggregateResult(options, results) {
               usage: usage2
             })
           );
+          metricResult.cost = metricCost;
+          caseEvaluationCost += metricCost;
+          evaluationCost += metricCost;
         }
       }
     }
+    result.usage = {
+      target: caseTargetUsage,
+      evaluation: caseEvaluationUsage,
+      total: Usage.add(caseTargetUsage, caseEvaluationUsage)
+    };
+    if (options.cost !== void 0) {
+      result.cost = {
+        currency: options.cost.currency,
+        target: caseTargetCost,
+        evaluation: caseEvaluationCost,
+        total: caseTargetCost + caseEvaluationCost
+      };
+    }
   }
   const usage = {
     target: targetUsage,
@@ -1647,7 +1911,12 @@ async function aggregateResult(options, results) {
   return aggregates;
 }
 async function resolveTargetUsage(options, testCase, output) {
-  const usage = options.targetUsage === void 0 ? usageFromOutput(output) : await options.targetUsage({ suiteName: options.name, case: testCase, output });
+  const usage = options.targetUsage === void 0 ? usageFromOutput(output) : await options.targetUsage({
+    suiteName: options.name,
+    case: testCase,
+    output,
+    signal: new AbortController().signal
+  });
   if (usage !== void 0) assertUsage(usage, `Target usage for case ${testCase.id}`);
   return usage;
 }
@@ -1671,6 +1940,25 @@ async function calculateCost(value) {
   return cost;
 }
 function validateSuiteOptions(options) {
+  if (options.name.trim().length === 0) {
+    throw new TypeError("Evaluation suite name must not be empty");
+  }
+  if (options.cases.length === 0) {
+    throw new TypeError("Evaluation suite must contain at least one case");
+  }
+  if (options.metrics.length === 0) {
+    throw new TypeError("Evaluation suite must contain at least one metric");
+  }
+  for (const testCase of options.cases) {
+    if (testCase.id.trim().length === 0) {
+      throw new TypeError("Evaluation case id must not be empty");
+    }
+  }
+  for (const metric of options.metrics) {
+    if (metric.name.trim().length === 0) {
+      throw new TypeError("Evaluation metric name must not be empty");
+    }
+  }
   assertUnique(
     options.cases.map((testCase) => testCase.id),
     "Evaluation case id"
@@ -1682,6 +1970,26 @@ function validateSuiteOptions(options) {
   if (options.cost !== void 0 && options.cost.currency.trim().length === 0) {
     throw new TypeError("Evaluation cost currency must not be empty");
   }
+  for (const [label, value] of [
+    ["concurrency", options.concurrency],
+    ["targetConcurrency", options.targetConcurrency],
+    ["metricConcurrency", options.metricConcurrency]
+  ]) {
+    if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
+      throw new RangeError(`Evaluation ${label} must be a positive integer`);
+    }
+  }
+  if (options.caseTimeoutMs !== void 0 && (!Number.isSafeInteger(options.caseTimeoutMs) || options.caseTimeoutMs < 1)) {
+    throw new RangeError("Evaluation caseTimeoutMs must be a positive integer");
+  }
+  if (options.shard !== void 0) {
+    if (!Number.isSafeInteger(options.shard.count) || options.shard.count < 1) {
+      throw new RangeError("Evaluation shard count must be a positive integer");
+    }
+    if (!Number.isSafeInteger(options.shard.index) || options.shard.index < 0 || options.shard.index >= options.shard.count) {
+      throw new RangeError("Evaluation shard index must be between 0 and count - 1");
+    }
+  }
 }
 function assertUnique(values, label) {
   const seen = /* @__PURE__ */ new Set();
@@ -1690,6 +1998,31 @@ function assertUnique(values, label) {
     seen.add(value);
   }
 }
+function selectCases(options) {
+  const requested = options.caseIds === void 0 ? void 0 : new Set(options.caseIds);
+  if (requested !== void 0) {
+    assertUnique([...options.caseIds], "Evaluation selected case id");
+    const available = new Set(options.cases.map((testCase) => testCase.id));
+    for (const id of requested) {
+      if (!available.has(id))
+        throw new TypeError(`Evaluation selected case id was not found: ${id}`);
+    }
+  }
+  const filtered = options.cases.filter(
+    (testCase, index) => (requested === void 0 || requested.has(testCase.id)) && (options.caseFilter === void 0 || options.caseFilter(testCase, index))
+  );
+  if (options.shard === void 0) return filtered;
+  return filtered.filter((_, index) => index % options.shard.count === options.shard.index);
+}
+async function notifyProgress(options, event) {
+  await options.onProgress?.(event);
+}
+function isAborted(signal) {
+  return signal?.aborted === true;
+}
+function suiteAbortReason(signal) {
+  return signal?.reason ?? new EvalAbortError();
+}
 
 // src/evals/cli.ts
 var EvalAssertionError = class extends Error {
@@ -1701,11 +2034,20 @@ ${mismatches.map((mismatch) => `- ${mismatch}`).join("\n")}`);
     this.mismatches = mismatches;
   }
 };
+function defineEvalExpectations(_suite, expectations) {
+  return expectations;
+}
+function formatEvalResult(result, options = {}) {
+  const format = options.format ?? "pretty";
+  if (format === "quiet") return "";
+  validatePrintOptions(options);
+  return format === "json" ? jsonResult(result, options) : prettyResult(result, options);
+}
 function printEvalResult(result, options = {}) {
   const format = options.format ?? "pretty";
   if (format === "quiet") return;
   const write = options.output?.stdout ?? ((text) => process.stdout.write(text));
-  write(`${format === "json" ? jsonResult(result) : prettyResult(result)}
+  write(`${formatEvalResult(result, options)}
 `);
 }
 function evalExitCode(result, expectations) {
@@ -1730,11 +2072,11 @@ function assertEvalOutcomes(result, expected) {
   }
 }
 async function runEvalCli(options) {
-  const { format, exitCode, expectations, output, ...suiteOptions } = options;
+  const { format, exitCode, expectations, output, maxValueLength, redact: redact2, ...suiteOptions } = options;
   const result = await runEvalSuite(
     suiteOptions
   );
-  printEvalResult(result, { format, output });
+  printEvalResult(result, { format, output, maxValueLength, redact: redact2 });
   const code = evalExitCode(result, expectations);
   const mismatches = expectationMismatches(result, expectations);
   if (mismatches.length > 0 && format !== "quiet") {
@@ -1749,7 +2091,7 @@ ${mismatches.map((value) => `- ${value}`).join("\n")}
   if (exitCode === true && code > currentExitCode) process.exitCode = code;
   return result;
 }
-function prettyResult(result) {
+function prettyResult(result, options) {
   const lines = [
     `${result.name} (${result.run.id})`,
     `Cases: ${totalsText(result.cases)}`,
@@ -1757,20 +2099,53 @@ function prettyResult(result) {
   ];
   for (const caseResult of result.results) {
     lines.push(``, `[${caseResult.outcome.toUpperCase()}] ${caseResult.case.id}`);
-    if (caseResult.output !== void 0) lines.push(`  output: ${displayValue(caseResult.output)}`);
-    if (caseResult.targetError !== void 0) {
-      lines.push(`  target error: ${errorText(caseResult.targetError)}`);
+    if (caseResult.targetStatus === "succeeded") {
+      lines.push(
+        `  output: ${displayValue(
+          redact(caseResult.output, options, { kind: "output", caseId: caseResult.case.id }),
+          options.maxValueLength
+        )}`
+      );
+    }
+    if (caseResult.targetStatus === "failed") {
+      lines.push(
+        `  target error: ${errorText(
+          redact(caseResult.targetError, options, { kind: "error", caseId: caseResult.case.id }),
+          options.maxValueLength
+        )}`
+      );
     }
     for (const metric of caseResult.metrics) {
       const parts = [`  - ${metric.metricName}: ${metric.outcome.outcome}`];
-      if (metric.outcome.score !== void 0)
-        parts.push(`score=${displayValue(metric.outcome.score)}`);
+      if (metric.outcome.score !== void 0) {
+        parts.push(
+          `score=${displayValue(
+            redact(metric.outcome.score, options, {
+              kind: "score",
+              caseId: caseResult.case.id,
+              metricName: metric.metricName
+            }),
+            options.maxValueLength
+          )}`
+        );
+      }
       if (metric.threshold !== void 0) parts.push(`threshold=${metric.threshold}`);
       if (metric.direction !== void 0) parts.push(`direction=${metric.direction}`);
       if (!metric.required) parts.push("optional");
       lines.push(parts.join(" | "));
       const explanation = metric.outcome.comment ?? (metric.outcome.outcome === "invalid" ? metric.outcome.reason : void 0);
-      if (explanation !== void 0) lines.push(`    ${explanation}`);
+      if (explanation !== void 0) {
+        lines.push(
+          `    ${displayValue(
+            redact(explanation, options, {
+              kind: "comment",
+              caseId: caseResult.case.id,
+              metricName: metric.metricName
+            }),
+            options.maxValueLength
+          )}`
+        );
+      }
     }
   }
   lines.push(
@@ -1785,14 +2160,15 @@ function prettyResult(result) {
   lines.push(`Duration: ${result.durationMs}ms`);
   return lines.join("\n");
 }
-function jsonResult(result) {
+function jsonResult(result, options) {
   return JSON.stringify(
-    result,
+    redactJsonResult(result, options),
     (_key, value) => {
       if (value instanceof Error) {
         return { name: value.name, message: value.message, stack: value.stack };
       }
       if (value instanceof RegExp) return String(value);
+      if (typeof value === "string") return truncate(value, options.maxValueLength);
       return value;
     },
     2
@@ -1885,16 +2261,107 @@ function requiredMetricCount(result, status) {
 function totalsText(totals) {
   return `${totals.total} total / ${totals.passed} pass / ${totals.failed} fail / ${totals.invalid} invalid`;
 }
-function displayValue(value) {
-  if (typeof value === "string") return value;
-  try {
-    return JSON.stringify(value);
-  } catch {
-    return String(value);
+function displayValue(value, maxValueLength) {
+  let text;
+  if (typeof value === "string") text = value;
+  else {
+    try {
+      text = JSON.stringify(value) ?? String(value);
+    } catch {
+      text = String(value);
+    }
   }
+  return truncate(text, maxValueLength);
+}
+function errorText(error, maxValueLength) {
+  return truncate(error instanceof Error ? error.message : displayValue(error), maxValueLength);
+}
+function truncate(value, maxValueLength) {
+  return maxValueLength !== void 0 && value.length > maxValueLength ? `${value.slice(0, maxValueLength)}\u2026` : value;
+}
+function redact(value, options, context) {
+  return options.redact === void 0 ? value : options.redact(value, context);
+}
+function redactJsonResult(result, options) {
+  return {
+    ...result,
+    run: {
+      ...result.run,
+      metadata: result.run.metadata === void 0 ? void 0 : redact(result.run.metadata, options, {
+        kind: "metadata"
+      })
+    },
+    results: result.results.map((caseResult) => ({
+      ...caseResult,
+      case: {
+        ...caseResult.case,
+        input: redact(caseResult.case.input, options, {
+          kind: "input",
+          caseId: caseResult.case.id
+        }),
+        expected: redact(caseResult.case.expected, options, {
+          kind: "expected",
+          caseId: caseResult.case.id
+        }),
+        context: redact(caseResult.case.context, options, {
+          kind: "context",
+          caseId: caseResult.case.id
+        }),
+        retrievalContext: redact(caseResult.case.retrievalContext, options, {
+          kind: "retrievalContext",
+          caseId: caseResult.case.id
+        }),
+        metadata: caseResult.case.metadata === void 0 ? void 0 : redact(caseResult.case.metadata, options, {
+          kind: "metadata",
+          caseId: caseResult.case.id
+        })
+      },
+      output: caseResult.targetStatus === "succeeded" ? redact(caseResult.output, options, { kind: "output", caseId: caseResult.case.id }) : void 0,
+      targetError: caseResult.targetStatus === "failed" ? redact(caseResult.targetError, options, {
+        kind: "error",
+        caseId: caseResult.case.id
+      }) : void 0,
+      metrics: caseResult.metrics.map((metric) => ({
+        ...metric,
+        reporterErrors: metric.reporterErrors.map(
+          (error) => redact(error, options, {
+            kind: "error",
+            caseId: caseResult.case.id,
+            metricName: metric.metricName
+          })
+        ),
+        outcome: redactOutcome(metric.outcome, options, caseResult.case.id, metric.metricName)
+      })),
+      scores: Object.fromEntries(
+        caseResult.metrics.map((metric) => [
+          metric.metricName,
+          redactOutcome(metric.outcome, options, caseResult.case.id, metric.metricName)
+        ])
+      )
+    })),
+    reporterErrors: result.reporterErrors.map((error) => redact(error, options, { kind: "error" }))
+  };
 }
-function errorText(error) {
-  return error instanceof Error ? error.message : displayValue(error);
+function redactOutcome(outcome, options, caseId, metricName) {
+  return {
+    ...outcome,
+    ...outcome.score === void 0 ? {} : { score: redact(outcome.score, options, { kind: "score", caseId, metricName }) },
+    ...outcome.comment === void 0 ? {} : { comment: redact(outcome.comment, options, { kind: "comment", caseId, metricName }) },
+    ...!("reason" in outcome) || outcome.reason === void 0 ? {} : { reason: redact(outcome.reason, options, { kind: "comment", caseId, metricName }) },
+    ...outcome.metadata === void 0 ? {} : {
+      metadata: redact(outcome.metadata, options, {
+        kind: "metadata",
+        caseId,
+        metricName
+      })
+    },
+    ...!("error" in outcome) || outcome.error === void 0 ? {} : { error: redact(outcome.error, options, { kind: "error", caseId, metricName }) }
+  };
+}
+function validatePrintOptions(options) {
+  if (options.maxValueLength !== void 0 && (!Number.isSafeInteger(options.maxValueLength) || options.maxValueLength < 1)) {
+    throw new RangeError("Evaluation maxValueLength must be a positive integer.");
+  }
 }
 
 // src/evals/metric.ts
@@ -1917,7 +2384,7 @@ function exactMatch(options = {}) {
       if (expected === void 0) {
         return EvalOutcome.invalid("No expected value provided for exact match.");
       }
-      const passed = stableComparable(actual) === stableComparable(expected);
+      const passed = evalValuesEqual(actual, expected);
       return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });
     }
   };
@@ -2078,12 +2545,13 @@ async function resolveOption(value, args) {
   return typeof value === "function" ? value(args) : value;
 }
 function semanticSimilarity(options) {
+  const threshold = unitInterval(options.threshold, "semanticSimilarity threshold");
   return {
     name: options.name ?? "semantic_similarity",
     required: options.required ?? true,
     dataType: "NUMERIC",
     direction: "higher_is_better",
-    threshold: options.threshold,
+    threshold,
     async evaluate(args) {
       const actual = await resolveActualText(options.actual, args);
       const expected = await resolveExpected(options.expected, args);
@@ -2098,11 +2566,12 @@ function semanticSimilarity(options) {
         embedText({ model: options.model, text: expected })
       ]);
       const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
-      return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
+      return score >= threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${threshold}.` });
     }
   };
 }
 function llmJudge(options) {
+  const retries = extractionRetries(options.retries);
   return {
     name: options.name ?? "llm_judge",
     required: options.required ?? true,
@@ -2113,24 +2582,27 @@ function llmJudge(options) {
           outputSchema: options.schema,
           instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema.",
           text: await resolveJudgePrompt(options.prompt, args),
-          retries: evalExtractionRetries(options.retries)
+          retries
         });
         return options.passes(result.output) ? EvalOutcome.pass(result.output, { usage: result.usage }) : EvalOutcome.fail(result.output, { usage: result.usage });
       } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
+        return EvalOutcome.fromError(error);
       }
     }
   };
 }
 function llmScore(options) {
   const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
+  if (criteria.trim().length === 0) throw new TypeError("llmScore criteria must not be empty.");
+  const threshold = unitInterval(options.threshold, "llmScore threshold");
+  const retries = extractionRetries(options.retries);
   return {
     name: options.name ?? "llm_score",
     required: options.required ?? true,
     dataType: "NUMERIC",
     projectScore: (score) => score.score,
     direction: "higher_is_better",
-    threshold: options.threshold,
+    threshold,
     async evaluate(args) {
       try {
         const result = await extract({
@@ -2144,7 +2616,7 @@ ${criteria}
 
 Return a score between 0 and 1 and brief feedback.`,
           text: await resolveJudgePrompt(options.prompt, args),
-          retries: evalExtractionRetries(options.retries)
+          retries
         });
         const score = result.output;
         if (score.score < 0 || score.score > 1) {
@@ -2153,38 +2625,53 @@ Return a score between 0 and 1 and brief feedback.`,
             usage: result.usage
           });
         }
-        return score.score >= options.threshold ? EvalOutcome.pass(score, { comment: score.feedback, usage: result.usage }) : EvalOutcome.fail(score, { comment: score.feedback, usage: result.usage });
+        return score.score >= threshold ? EvalOutcome.pass(score, { comment: score.feedback, usage: result.usage }) : EvalOutcome.fail(score, { comment: score.feedback, usage: result.usage });
       } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
+        return EvalOutcome.fromError(error);
       }
     }
   };
 }
-function evalExtractionRetries(retries) {
-  const retryCount = Math.max(0, Math.trunc(retries ?? 0));
+function extractionRetries(retries) {
+  const retryCount = retries ?? 0;
+  if (!Number.isInteger(retryCount) || retryCount < 0) {
+    throw new RangeError("Eval metric retries must be a non-negative integer.");
+  }
   if (retryCount === 0) {
     return void 0;
   }
   return { maxAttempts: retryCount + 1 };
 }
+function unitInterval(value, label) {
+  if (!Number.isFinite(value) || value < 0 || value > 1) {
+    throw new RangeError(`${label} must be between 0 and 1.`);
+  }
+  return value;
+}
 
 // src/evals/suite.ts
 function defineEvalCases(cases) {
   return cases;
 }
-function defineEvalSuite(options) {
-  if (options !== void 0) return options;
+function createEvalTypes() {
   return {
     defineMetric(metric) {
       return metric;
     }
   };
 }
+function defineEvalSuite(options) {
+  if (options !== void 0) return options;
+  return createEvalTypes();
+}
 export {
   AgentEvalSuspensionError,
+  EvalAbortError,
   EvalAssertionError,
+  EvalFailFastError,
   EvalOutcome,
   EvalReporterDispatchError,
+  EvalTimeoutError,
   abstention,
   agentEvalTarget,
   answerRelevancy,
@@ -2193,14 +2680,17 @@ export {
   contains,
   containsAll,
   containsAny,
+  createEvalTypes,
   defaultEvalTraceSelector,
   defineEvalCases,
+  defineEvalExpectations,
   defineEvalSuite,
   defineMetric,
   doesNotMatch,
   evalExitCode,
   exactMatch,
   faithfulness,
+  formatEvalResult,
   gEval,
   hallucination,
   jsonCorrectness,
@@ -2217,6 +2707,7 @@ export {
   resolveEvalTraceRef,
   runEvalCli,
   runEvalSuite,
+  selectEvalCaseIds,
   selectPromptOutput,
   semanticSimilarity,
   summarization,