@anvia/core 0.24.0 → 0.25.0

package/dist/evals/index.js

@@ -1,16 +1,22 @@
 import {
   ExtractorBuilder
-} from "../chunk-YLMDBQO4.js";
-import "../chunk-JI5YZWNT.js";
-import "../chunk-DALYP4PX.js";
-import "../chunk-BALBBVI3.js";
+} from "../chunk-KYEBWN3E.js";
+import "../chunk-4HWN7734.js";
+import "../chunk-YEBIVZM2.js";
+import "../chunk-VXONXP2G.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-HYUYZEAJ.js";
-import "../chunk-MELYDKWQ.js";
+import "../chunk-2NDQHV7V.js";
+import "../chunk-4BGN6PYF.js";
 import "../chunk-2ODTMRHP.js";
 import "../chunk-UQX6EXWG.js";
 import "../chunk-MRQLEK2B.js";
+import "../chunk-Q25IWLBT.js";
+import "../chunk-MC3CCKEB.js";
+import {
+  Usage
+} from "../chunk-ADH7NNCS.js";
+import "../chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText
@@ -18,12 +24,6 @@ import {
 import {
   mapWithConcurrency
 } from "../chunk-OIMLU4SF.js";
-import "../chunk-Q25IWLBT.js";
-import "../chunk-MC3CCKEB.js";
-import {
-  Usage
-} from "../chunk-ADH7NNCS.js";
-import "../chunk-WQKHFADH.js";
 import "../chunk-CWUJUSOS.js";
 
 // src/evals/advanced-metrics.ts
@@ -100,6 +100,9 @@ var EvalOutcome = {
     if (options.metadata !== void 0) {
       outcome.metadata = options.metadata;
     }
+    if (options.usage !== void 0) {
+      outcome.usage = options.usage;
+    }
     return outcome;
   },
   fail(score, options = {}) {
@@ -115,6 +118,9 @@ var EvalOutcome = {
     if (options.metadata !== void 0) {
       outcome.metadata = options.metadata;
     }
+    if (options.usage !== void 0) {
+      outcome.usage = options.usage;
+    }
     return outcome;
   },
   invalid(reason, options = {}) {
@@ -131,11 +137,20 @@ var EvalOutcome = {
     if (options.metadata !== void 0) {
       outcome.metadata = options.metadata;
     }
+    if (options.usage !== void 0) {
+      outcome.usage = options.usage;
+    }
     return outcome;
   }
 };
 
 // src/evals/selectors.ts
+function selectPromptOutput(args) {
+  if (typeof args.output !== "object" || args.output === null || !("output" in args.output) || typeof args.output.output !== "string") {
+    throw new TypeError("selectPromptOutput requires an output object with a string output field.");
+  }
+  return args.output.output;
+}
 async function resolveActual(selector, args) {
   return selector === void 0 ? defaultOutputValue(args.output) : selector(args);
 }
@@ -188,9 +203,14 @@ var binaryVerdictSchema = z.object({
   reason: z.string()
 });
 var reasonSchema = z.object({ reason: z.string() });
+var abstentionJudgmentSchema = z.object({
+  behavior: z.enum(["abstention", "confident_answer"]),
+  grounded: z.boolean(),
+  reason: z.string()
+});
 function answerRelevancy(options) {
   const config = metricConfig(options, "answer_relevancy");
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const input = await resolveInput(options.input, args);
       const actual = await resolveActualText(options.actual, args);
@@ -245,7 +265,7 @@ function promptAlignment(options) {
     throw new TypeError("promptAlignment requires at least one prompt instruction.");
   }
   const config = metricConfig(options, "prompt_alignment");
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const input = await resolveInput(options.input, args);
       const actual = await resolveActualText(options.actual, args);
@@ -285,56 +305,65 @@ function jsonCorrectness(options) {
   const threshold = validateThreshold(options.threshold ?? 0.5);
   const retries = validateRetries(options.retries ?? 0);
   const includeReason = options.includeReason ?? true;
-  return numericMetric(options.name ?? "json_correctness", async (args) => {
-    try {
-      const actual = await resolveActualText(options.actual, args);
-      let parsed;
-      let validationError;
+  const strictMode = options.strictMode ?? true;
+  return numericMetric(
+    options.name ?? "json_correctness",
+    {
+      threshold: strictMode ? 1 : threshold,
+      required: options.required ?? true
+    },
+    "higher_is_better",
+    async (args) => {
       try {
-        parsed = JSON.parse(actual);
-        const result = options.schema.safeParse(parsed);
-        if (!result.success) {
-          validationError = z.prettifyError(result.error);
+        const actual = await resolveActualText(options.actual, args);
+        let parsed;
+        let validationError;
+        try {
+          parsed = JSON.parse(actual);
+          const result = options.schema.safeParse(parsed);
+          if (!result.success) {
+            validationError = z.prettifyError(result.error);
+          }
+        } catch (error) {
+          validationError = errorMessage(error);
         }
-      } catch (error) {
-        validationError = errorMessage(error);
-      }
-      const score = validationError === void 0 ? 1 : 0;
-      let comment;
-      let usage = Usage.empty();
-      if (includeReason) {
-        if (score === 1) {
-          comment = "The generated JSON is syntactically valid and matches the expected schema.";
-        } else if (options.model === void 0) {
-          comment = validationError;
-        } else {
-          const reasonResult = await runJudge({
-            model: options.model,
-            schema: reasonSchema,
-            instructions: "Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.",
-            prompt: jsonPrompt({ actual, validationError }),
-            retries
-          });
-          comment = reasonResult.data.reason;
-          usage = reasonResult.usage;
+        const score = validationError === void 0 ? 1 : 0;
+        let comment;
+        let usage = Usage.empty();
+        if (includeReason) {
+          if (score === 1) {
+            comment = "The generated JSON is syntactically valid and matches the expected schema.";
+          } else if (options.model === void 0) {
+            comment = validationError;
+          } else {
+            const reasonResult = await runJudge({
+              model: options.model,
+              schema: reasonSchema,
+              instructions: "Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.",
+              prompt: jsonPrompt({ actual, validationError }),
+              retries
+            });
+            comment = reasonResult.data.reason;
+            usage = reasonResult.usage;
+          }
         }
+        return higherOutcome({
+          score,
+          threshold,
+          strictMode,
+          comment,
+          details: validationError === void 0 ? {} : { validationError },
+          usage
+        });
+      } catch (error) {
+        return EvalOutcome.invalid(errorMessage(error));
       }
-      return higherOutcome({
-        score,
-        threshold,
-        strictMode: options.strictMode ?? true,
-        comment,
-        details: validationError === void 0 ? {} : { validationError },
-        usage
-      });
-    } catch (error) {
-      return EvalOutcome.invalid(errorMessage(error));
     }
-  });
+  );
 }
 function hallucination(options) {
   const config = metricConfig(options, "hallucination");
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "lower_is_better", async (args) => {
     try {
       const actual = await resolveActualText(options.actual, args);
       const context = await resolveStringList(options.context, args.case.context, args, "context");
@@ -376,7 +405,7 @@ function faithfulness(options) {
     options.truthsExtractionLimit,
     "truthsExtractionLimit"
   );
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const actual = await resolveActualText(options.actual, args);
       const retrievalContext = await resolveStringList(
@@ -445,6 +474,70 @@ ${actual}`,
     }
   });
 }
+function abstention(options) {
+  const retries = validateRetries(options.retries ?? 0);
+  return {
+    name: options.name ?? "abstention",
+    required: options.required ?? true,
+    dataType: "CATEGORICAL",
+    async evaluate(args) {
+      try {
+        const actual = await resolveActualText(options.actual, args);
+        const shouldAbstain = await resolveExpected(options.shouldAbstain, args);
+        if (typeof shouldAbstain !== "boolean") {
+          return EvalOutcome.invalid("abstention shouldAbstain must resolve to a boolean.");
+        }
+        const context = await resolveAbstentionContext(options.context, args);
+        if (!shouldAbstain && context.length === 0) {
+          return EvalOutcome.invalid(
+            "abstention context must be non-empty when shouldAbstain is false."
+          );
+        }
+        const judgment = await runJudge({
+          model: options.model,
+          schema: abstentionJudgmentSchema,
+          instructions: "Classify whether the answer abstains or gives a confident answer. For a confident answer, grounded is true only when every substantive factual claim is supported by the supplied context. For an abstention, set grounded to false. Return a concise evidence-based reason.",
+          prompt: jsonPrompt({ actual, context }),
+          retries
+        });
+        const category = abstentionCategory(
+          shouldAbstain,
+          judgment.data.behavior,
+          judgment.data.grounded
+        );
+        const outcomeOptions = {
+          comment: options.includeReason === false ? void 0 : judgment.data.reason,
+          metadata: evaluationMetadata(
+            {
+              behavior: judgment.data.behavior,
+              grounded: judgment.data.grounded,
+              shouldAbstain
+            },
+            judgment.usage
+          ),
+          usage: judgment.usage
+        };
+        return category === "correct_abstention" || category === "correct_grounded_answer" ? EvalOutcome.pass(category, outcomeOptions) : EvalOutcome.fail(category, outcomeOptions);
+      } catch (error) {
+        return EvalOutcome.invalid(errorMessage(error));
+      }
+    }
+  };
+}
+async function resolveAbstentionContext(selectorOrValue, args) {
+  const context = selectorOrValue === void 0 ? args.case.retrievalContext ?? [] : typeof selectorOrValue === "function" ? await selectorOrValue(args) : selectorOrValue;
+  if (!Array.isArray(context) || context.some((value) => typeof value !== "string")) {
+    throw new TypeError("abstention context must be an array of strings.");
+  }
+  return context;
+}
+function abstentionCategory(shouldAbstain, behavior, grounded) {
+  if (behavior === "abstention") {
+    return shouldAbstain ? "correct_abstention" : "unnecessary_abstention";
+  }
+  if (shouldAbstain || !grounded) return "unsupported_confident_answer";
+  return "correct_grounded_answer";
+}
 function summarization(options) {
   const config = metricConfig(options, "summarization");
   const questionCount = validatePositiveInteger(options.questionCount ?? 5, "questionCount");
@@ -453,7 +546,7 @@ function summarization(options) {
     "truthsExtractionLimit"
   );
   const suppliedQuestions = options.assessmentQuestions !== void 0 && options.assessmentQuestions.length > 0 ? [...options.assessmentQuestions] : void 0;
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const input = await resolveInput(options.input, args);
       const actual = await resolveActualText(options.actual, args);
@@ -615,7 +708,7 @@ function gEval(options) {
     generatedUsageClaimed = true;
     return { steps: result.data.steps, usage };
   }
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const parameters = await resolveGEvalParameters(options, args);
       const stepsResult = await resolveSteps();
@@ -662,7 +755,7 @@ function turnRelevancy(options) {
   const config = metricConfig(options, "turn_relevancy");
   const windowSize = validatePositiveInteger(options.windowSize ?? 10, "windowSize");
   const concurrency = validatePositiveInteger(options.concurrency ?? 4, "concurrency");
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const turns = await resolveTurns(options.turns, args);
       const interactions = unitInteractions(turns);
@@ -710,7 +803,7 @@ function turnRelevancy(options) {
 function knowledgeRetention(options) {
   const config = metricConfig(options, "knowledge_retention");
   const concurrency = validatePositiveInteger(options.concurrency ?? 4, "concurrency");
-  return numericMetric(config.name, async (args) => {
+  return numericMetric(config.name, config, "higher_is_better", async (args) => {
     try {
       const turns = await resolveTurns(options.turns, args);
       const userTurns = turns.map((turn, index) => ({ turn, index })).filter((entry) => entry.turn.role === "user");
@@ -779,8 +872,15 @@ function knowledgeRetention(options) {
     }
   });
 }
-function numericMetric(name, evaluate) {
-  return { name, dataType: "NUMERIC", evaluate };
+function numericMetric(name, config, direction, evaluate) {
+  return {
+    name,
+    required: config.required,
+    direction,
+    threshold: config.strictMode === true ? direction === "higher_is_better" ? 1 : 0 : config.threshold,
+    dataType: "NUMERIC",
+    evaluate
+  };
 }
 function metricConfig(options, defaultName) {
   return {
@@ -788,7 +888,8 @@ function metricConfig(options, defaultName) {
     threshold: validateThreshold(options.threshold ?? 0.5),
     strictMode: options.strictMode ?? false,
     includeReason: options.includeReason ?? true,
-    retries: validateRetries(options.retries ?? 0)
+    retries: validateRetries(options.retries ?? 0),
+    required: options.required ?? true
   };
 }
 function higherOutcome(args) {
@@ -796,6 +897,7 @@ function higherOutcome(args) {
   const threshold = args.strictMode ? 1 : args.threshold;
   const options = {
     comment: args.comment,
+    usage: args.usage,
     metadata: evaluationMetadata(
       {
         ...args.details,
@@ -813,6 +915,7 @@ function lowerOutcome(args) {
   const threshold = args.strictMode ? 0 : args.threshold;
   const options = {
     comment: args.comment,
+    usage: args.usage,
     metadata: evaluationMetadata(
       {
         ...args.details,
@@ -1021,122 +1124,9 @@ function agentEvalTarget(agent, options = {}) {
   };
 }
 
-// src/evals/metric.ts
-function defineMetric(metric) {
-  return metric;
-}
-
-// src/evals/metrics.ts
-import { z as z2 } from "zod";
-function exactMatch(options = {}) {
-  return {
-    name: options.name ?? "exact_match",
-    async evaluate(args) {
-      const actual = await resolveActual(options.actual, args);
-      const expected = await resolveExpected(options.expected, args);
-      if (expected === void 0) {
-        return EvalOutcome.invalid("No expected value provided for exact match.");
-      }
-      const passed = stableComparable(actual) === stableComparable(expected);
-      return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });
-    }
-  };
-}
-function contains(options = {}) {
-  return {
-    name: options.name ?? "contains",
-    async evaluate(args) {
-      const actual = await resolveActualText(options.actual, args);
-      const expected = await resolveExpected(options.expected, args);
-      if (expected === void 0) {
-        return EvalOutcome.invalid("No expected value provided for contains.");
-      }
-      if (typeof expected !== "string" && !(expected instanceof RegExp)) {
-        return EvalOutcome.invalid("Contains expected value must be a string or RegExp.");
-      }
-      const passed = expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);
-      return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });
-    }
-  };
-}
-function regexMatches(pattern, text) {
-  pattern.lastIndex = 0;
-  const matched = pattern.test(text);
-  pattern.lastIndex = 0;
-  return matched;
-}
-function semanticSimilarity(options) {
-  return {
-    name: options.name ?? "semantic_similarity",
-    async evaluate(args) {
-      const actual = await resolveActualText(options.actual, args);
-      const expected = await resolveExpected(options.expected, args);
-      if (expected === void 0) {
-        return EvalOutcome.invalid("No expected value provided for semantic similarity.");
-      }
-      if (typeof expected !== "string") {
-        return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
-      }
-      const [actualEmbedding, expectedEmbedding] = await Promise.all([
-        embedText(options.model, actual),
-        embedText(options.model, expected)
-      ]);
-      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
-      return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
-    }
-  };
-}
-function llmJudge(options) {
-  const extractor = new ExtractorBuilder(options.model, options.schema).instructions(
-    options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema."
-  ).retries(options.retries ?? 0).build();
-  return {
-    name: options.name ?? "llm_judge",
-    async evaluate(args) {
-      try {
-        const judgment = await extractor.extract(await resolveJudgePrompt(options.prompt, args));
-        return options.passes(judgment) ? EvalOutcome.pass(judgment) : EvalOutcome.fail(judgment);
-      } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
-      }
-    }
-  };
-}
-function llmScore(options) {
-  const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
-  const extractor = new ExtractorBuilder(
-    options.model,
-    z2.object({
-      score: z2.number(),
-      feedback: z2.string()
-    })
-  ).instructions(
-    options.instructions ?? `Score the eval case against these criteria:
-${criteria}
-
-Return a score between 0 and 1 and brief feedback.`
-  ).retries(options.retries ?? 0).build();
-  return {
-    name: options.name ?? "llm_score",
-    async evaluate(args) {
-      try {
-        const score = await extractor.extract(await resolveJudgePrompt(options.prompt, args));
-        if (score.score < 0 || score.score > 1) {
-          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
-            score
-          });
-        }
-        return score.score >= options.threshold ? EvalOutcome.pass(score, { comment: score.feedback }) : EvalOutcome.fail(score, { comment: score.feedback });
-      } catch (error) {
-        return EvalOutcome.invalid(errorMessage(error));
-      }
-    }
-  };
-}
-
 // src/evals/reporting.ts
-function projectEvalOutcome(outcome, dataType) {
-  const value = projectScoreValue(outcome, dataType);
+function projectEvalOutcome(outcome, dataType, projectScore) {
+  const value = projectScoreValue(outcome, dataType, projectScore);
   const projection = {
     outcome: outcome.outcome,
     value,
@@ -1158,8 +1148,13 @@ function defaultEvalTraceSelector(args) {
     metadata: args.case.metadata
   });
 }
-function projectScoreValue(outcome, dataType) {
+function projectScoreValue(outcome, dataType, projectScore) {
   const score = outcome.score;
+  if (score !== void 0 && projectScore !== void 0) {
+    const projected = projectScore(score);
+    if (typeof projected === "boolean") return projected ? 1 : 0;
+    return projected;
+  }
   if (dataType === "CATEGORICAL") {
     if (typeof score === "string") return score;
     if (typeof score === "number") return String(score);
@@ -1211,6 +1206,7 @@ function readTraceRef(value) {
 
 // src/evals/runner.ts
 async function runEvalSuite(options) {
+  validateSuiteOptions(options);
   const startedAtMs = Date.now();
   const run = resolveRun(options, startedAtMs);
   const reporters = options.reporters ?? [];
@@ -1238,8 +1234,10 @@ async function runEvalSuite(options) {
     throw error;
   }
   let results;
+  let aggregates;
   try {
     results = await runEvalCases(options, run);
+    aggregates = await aggregateResult(options, results);
   } catch (error) {
     await notifyRunEnd(reporters, {
       ...lifecycle,
@@ -1250,13 +1248,15 @@ async function runEvalSuite(options) {
     });
     throw error;
   }
-  const counts = countOutcomes(results);
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const result = {
     name: options.name,
     run: { ...run, completedAt },
     results,
-    ...counts,
+    metrics: aggregates.metrics,
+    cases: aggregates.cases,
+    usage: aggregates.usage,
+    ...aggregates.cost === void 0 ? {} : { cost: aggregates.cost },
     durationMs: Date.now() - startedAtMs,
     reporterErrors
   };
@@ -1268,7 +1268,10 @@ async function runEvalSuite(options) {
         status: "completed",
         completedAt,
         durationMs: result.durationMs,
-        ...counts
+        metrics: result.metrics,
+        cases: result.cases,
+        usage: result.usage,
+        ...result.cost === void 0 ? {} : { cost: result.cost }
       },
       options.failOnReporterError === true
     )
@@ -1326,11 +1329,24 @@ async function runEvalCase(options, testCase, run) {
       reporters: options.reporters ?? [],
       failOnReporterError: options.failOnReporterError === true
     });
-    metrics.push({ metricName: metric.name, outcome, reporterErrors });
+    const metricResult = {
+      metricName: metric.name,
+      required: metric.required ?? true,
+      outcome,
+      reporterErrors
+    };
+    if (metric.direction !== void 0) metricResult.direction = metric.direction;
+    if (metric.threshold !== void 0) metricResult.threshold = metric.threshold;
+    metrics.push(metricResult);
   }
+  const scores = Object.fromEntries(
+    metrics.map((metric) => [metric.metricName, metric.outcome])
+  );
   const result = {
     case: testCase,
-    metrics
+    outcome: caseOutcome(targetError, metrics),
+    metrics,
+    scores
   };
   if (output !== void 0) {
     result.output = output;
@@ -1436,26 +1452,648 @@ async function notifyRunEnd(reporters, args, failOnReporterError = false) {
   }
   return errors;
 }
-function countOutcomes(results) {
-  let passed = 0;
-  let failed = 0;
-  let invalid = 0;
+function countMetricOutcomes(results) {
+  const totals = emptyTotals();
   for (const result of results) {
     for (const metric of result.metrics) {
-      if (metric.outcome.outcome === "pass") passed += 1;
-      if (metric.outcome.outcome === "fail") failed += 1;
-      if (metric.outcome.outcome === "invalid") invalid += 1;
+      totals.total += 1;
+      totals[statusKey(metric.outcome.outcome)] += 1;
+    }
+  }
+  return totals;
+}
+function countCaseOutcomes(results) {
+  const totals = emptyTotals();
+  for (const result of results) {
+    totals.total += 1;
+    totals[statusKey(result.outcome)] += 1;
+  }
+  return totals;
+}
+function emptyTotals() {
+  return { total: 0, passed: 0, failed: 0, invalid: 0 };
+}
+function statusKey(status) {
+  if (status === "pass") return "passed";
+  if (status === "fail") return "failed";
+  return "invalid";
+}
+function caseOutcome(targetError, metrics) {
+  if (targetError !== void 0) return "invalid";
+  const required = metrics.filter((metric) => metric.required);
+  if (required.some((metric) => metric.outcome.outcome === "invalid")) return "invalid";
+  if (required.some((metric) => metric.outcome.outcome === "fail")) return "fail";
+  return "pass";
+}
+async function aggregateResult(options, results) {
+  let targetUsage = Usage.empty();
+  let evaluationUsage = Usage.empty();
+  let targetCost = 0;
+  let evaluationCost = 0;
+  for (const result of results) {
+    if (result.output !== void 0) {
+      const usage2 = await resolveTargetUsage(options, result.case, result.output);
+      if (usage2 !== void 0) {
+        targetUsage = Usage.add(targetUsage, usage2);
+        if (options.cost !== void 0) {
+          targetCost += await calculateCost(
+            options.cost.calculate({
+              kind: "target",
+              suiteName: options.name,
+              case: result.case,
+              output: result.output,
+              usage: usage2
+            })
+          );
+        }
+      }
+    }
+    for (const metricResult of result.metrics) {
+      const usage2 = metricResult.outcome.usage;
+      if (usage2 === void 0) continue;
+      assertUsage(usage2, `Evaluation usage for metric ${metricResult.metricName}`);
+      evaluationUsage = Usage.add(evaluationUsage, usage2);
+      if (options.cost !== void 0 && result.output !== void 0) {
+        const metric = options.metrics.find(
+          (candidate) => candidate.name === metricResult.metricName
+        );
+        if (metric !== void 0) {
+          evaluationCost += await calculateCost(
+            options.cost.calculate({
+              kind: "evaluation",
+              suiteName: options.name,
+              case: result.case,
+              output: result.output,
+              metric,
+              usage: usage2
+            })
+          );
+        }
+      }
+    }
+  }
+  const usage = {
+    target: targetUsage,
+    evaluation: evaluationUsage,
+    total: Usage.add(targetUsage, evaluationUsage)
+  };
+  const aggregates = {
+    metrics: countMetricOutcomes(results),
+    cases: countCaseOutcomes(results),
+    usage
+  };
+  if (options.cost !== void 0) {
+    aggregates.cost = {
+      currency: options.cost.currency,
+      target: targetCost,
+      evaluation: evaluationCost,
+      total: targetCost + evaluationCost
+    };
+  }
+  return aggregates;
+}
+async function resolveTargetUsage(options, testCase, output) {
+  const usage = options.targetUsage === void 0 ? usageFromOutput(output) : await options.targetUsage({ suiteName: options.name, case: testCase, output });
+  if (usage !== void 0) assertUsage(usage, `Target usage for case ${testCase.id}`);
+  return usage;
+}
+function usageFromOutput(output) {
+  if (typeof output !== "object" || output === null || !("usage" in output)) return void 0;
+  return output.usage;
+}
+function assertUsage(usage, label) {
+  for (const [key, value] of Object.entries(usage)) {
+    if (key === "details") continue;
+    if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+      throw new TypeError(`${label} must contain finite, non-negative token counts`);
+    }
+  }
+}
+async function calculateCost(value) {
+  const cost = await value;
+  if (!Number.isFinite(cost) || cost < 0) {
+    throw new TypeError("Evaluation cost calculator must return a finite, non-negative number");
+  }
+  return cost;
+}
+function validateSuiteOptions(options) {
+  assertUnique(
+    options.cases.map((testCase) => testCase.id),
+    "Evaluation case id"
+  );
+  assertUnique(
+    options.metrics.map((metric) => metric.name),
+    "Evaluation metric name"
+  );
+  if (options.cost !== void 0 && options.cost.currency.trim().length === 0) {
+    throw new TypeError("Evaluation cost currency must not be empty");
+  }
+}
+function assertUnique(values, label) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (seen.has(value)) throw new TypeError(`${label} must be unique: ${value}`);
+    seen.add(value);
+  }
+}
+
+// src/evals/cli.ts
+var EvalAssertionError = class extends Error {
+  mismatches;
+  constructor(message, mismatches) {
+    super(`${message}
+${mismatches.map((mismatch) => `- ${mismatch}`).join("\n")}`);
+    this.name = "EvalAssertionError";
+    this.mismatches = mismatches;
+  }
+};
+function printEvalResult(result, options = {}) {
+  const format = options.format ?? "pretty";
+  if (format === "quiet") return;
+  const write = options.output?.stdout ?? ((text) => process.stdout.write(text));
+  write(`${format === "json" ? jsonResult(result) : prettyResult(result)}
+`);
+}
+function evalExitCode(result, expectations) {
+  const hasExpectations = expectations?.totals !== void 0 || expectations?.outcomes !== void 0;
+  const mismatches = hasExpectations ? expectationMismatches(result, expectations) : [];
+  if (hasExpectations && mismatches.length === 0) return 0;
+  if (hasExpectations && hasUnexpectedInvalid(result, expectations)) return 2;
+  if (hasExpectations) return 1;
+  if (requiredMetricCount(result, "invalid") > 0) return 2;
+  return requiredMetricCount(result, "fail") > 0 ? 1 : 0;
+}
+function assertEvalTotals(result, expected) {
+  const mismatches = totalMismatches(result, expected);
+  if (mismatches.length > 0) {
+    throw new EvalAssertionError("Evaluation totals did not match expectations.", mismatches);
+  }
+}
+function assertEvalOutcomes(result, expected) {
+  const mismatches = outcomeMismatches(result, expected);
+  if (mismatches.length > 0) {
+    throw new EvalAssertionError("Evaluation outcomes did not match expectations.", mismatches);
+  }
+}
+async function runEvalCli(options) {
+  const { format, exitCode, expectations, output, ...suiteOptions } = options;
+  const result = await runEvalSuite(
+    suiteOptions
+  );
+  printEvalResult(result, { format, output });
+  const code = evalExitCode(result, expectations);
+  const mismatches = expectationMismatches(result, expectations);
+  if (mismatches.length > 0 && format !== "quiet") {
+    const write = output?.stderr ?? ((text) => process.stderr.write(text));
+    write(
+      `Evaluation expectation mismatches:
+${mismatches.map((value) => `- ${value}`).join("\n")}
+`
+    );
+  }
+  const currentExitCode = typeof process.exitCode === "number" ? process.exitCode : Number(process.exitCode ?? 0);
+  if (exitCode === true && code > currentExitCode) process.exitCode = code;
+  return result;
+}
+function prettyResult(result) {
+  const lines = [
+    `${result.name} (${result.run.id})`,
+    `Cases: ${totalsText(result.cases)}`,
+    `Metrics: ${totalsText(result.metrics)}`
+  ];
+  for (const caseResult of result.results) {
+    lines.push(``, `[${caseResult.outcome.toUpperCase()}] ${caseResult.case.id}`);
+    if (caseResult.output !== void 0) lines.push(`  output: ${displayValue(caseResult.output)}`);
+    if (caseResult.targetError !== void 0) {
+      lines.push(`  target error: ${errorText(caseResult.targetError)}`);
+    }
+    for (const metric of caseResult.metrics) {
+      const parts = [`  - ${metric.metricName}: ${metric.outcome.outcome}`];
+      if (metric.outcome.score !== void 0)
+        parts.push(`score=${displayValue(metric.outcome.score)}`);
+      if (metric.threshold !== void 0) parts.push(`threshold=${metric.threshold}`);
+      if (metric.direction !== void 0) parts.push(`direction=${metric.direction}`);
+      if (!metric.required) parts.push("optional");
+      lines.push(parts.join(" | "));
+      const explanation = metric.outcome.comment ?? (metric.outcome.outcome === "invalid" ? metric.outcome.reason : void 0);
+      if (explanation !== void 0) lines.push(`    ${explanation}`);
+    }
+  }
+  lines.push(
+    ``,
+    `Usage: target=${result.usage.target.totalTokens} evaluation=${result.usage.evaluation.totalTokens} total=${result.usage.total.totalTokens} tokens`
+  );
+  if (result.cost !== void 0) {
+    lines.push(
+      `Cost: target=${result.cost.target} evaluation=${result.cost.evaluation} total=${result.cost.total} ${result.cost.currency}`
+    );
+  }
+  lines.push(`Duration: ${result.durationMs}ms`);
+  return lines.join("\n");
+}
+function jsonResult(result) {
+  return JSON.stringify(
+    result,
+    (_key, value) => {
+      if (value instanceof Error) {
+        return { name: value.name, message: value.message, stack: value.stack };
+      }
+      if (value instanceof RegExp) return String(value);
+      return value;
+    },
+    2
+  );
+}
+function expectationMismatches(result, expectations) {
+  if (expectations === void 0) return [];
+  return [
+    ...expectations.totals === void 0 ? [] : totalMismatches(result, expectations.totals),
+    ...expectations.outcomes === void 0 ? [] : outcomeMismatches(result, expectations.outcomes)
+  ];
+}
+function totalMismatches(result, expected) {
+  const directMetrics = {
+    ...expected.total === void 0 ? {} : { total: expected.total },
+    ...expected.passed === void 0 ? {} : { passed: expected.passed },
+    ...expected.failed === void 0 ? {} : { failed: expected.failed },
+    ...expected.invalid === void 0 ? {} : { invalid: expected.invalid }
+  };
+  return [
+    ...totalsGroupMismatches("metrics", result.metrics, {
+      ...directMetrics,
+      ...expected.metrics
+    }),
+    ...totalsGroupMismatches("cases", result.cases, expected.cases)
+  ];
+}
+function totalsGroupMismatches(label, actual, expected) {
+  if (expected === void 0) return [];
+  const mismatches = [];
+  for (const key of ["total", "passed", "failed", "invalid"]) {
+    if (expected[key] !== void 0 && actual[key] !== expected[key]) {
+      mismatches.push(`${label}.${key}: expected ${expected[key]}, received ${actual[key]}`);
     }
   }
-  return { passed, failed, invalid };
+  return mismatches;
+}
+function outcomeMismatches(result, expected) {
+  const mismatches = [];
+  const actualCases = new Map(result.results.map((caseResult) => [caseResult.case.id, caseResult]));
+  for (const [caseId, metrics] of Object.entries(expected)) {
+    const caseResult = actualCases.get(caseId);
+    if (caseResult === void 0) {
+      mismatches.push(`${caseId}: expected case was not present`);
+      continue;
+    }
+    for (const [metricName, expectedOutcome] of Object.entries(metrics)) {
+      const metric = caseResult.metrics.find((candidate) => candidate.metricName === metricName);
+      if (metric === void 0) {
+        mismatches.push(`${caseId}.${metricName}: expected metric was not present`);
+      } else if (metric.outcome.outcome !== expectedOutcome) {
+        mismatches.push(
+          `${caseId}.${metricName}: expected ${expectedOutcome}, received ${metric.outcome.outcome}`
+        );
+      }
+    }
+  }
+  for (const caseResult of result.results) {
+    for (const metric of caseResult.metrics) {
+      if (!metric.required) continue;
+      const expectedOutcome = expected[caseResult.case.id]?.[metric.metricName] ?? "pass";
+      if (metric.outcome.outcome !== expectedOutcome) {
+        const message = `${caseResult.case.id}.${metric.metricName}: expected ${expectedOutcome}, received ${metric.outcome.outcome}`;
+        if (!mismatches.includes(message)) mismatches.push(message);
+      }
+    }
+  }
+  return mismatches;
+}
+function hasUnexpectedInvalid(result, expectations) {
+  if (expectations?.outcomes !== void 0) {
+    return result.results.some(
+      (caseResult) => caseResult.metrics.some(
+        (metric) => metric.required && metric.outcome.outcome === "invalid" && expectations.outcomes?.[caseResult.case.id]?.[metric.metricName] !== "invalid"
+      )
+    );
+  }
+  const expectedInvalid = expectations?.totals?.metrics?.invalid ?? expectations?.totals?.invalid;
+  return result.metrics.invalid > (expectedInvalid ?? 0);
+}
+function requiredMetricCount(result, status) {
+  return result.results.reduce(
+    (total, caseResult) => total + caseResult.metrics.filter((metric) => metric.required && metric.outcome.outcome === status).length,
+    0
+  );
+}
+function totalsText(totals) {
+  return `${totals.total} total / ${totals.passed} pass / ${totals.failed} fail / ${totals.invalid} invalid`;
+}
+function displayValue(value) {
+  if (typeof value === "string") return value;
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+function errorText(error) {
+  return error instanceof Error ? error.message : displayValue(error);
+}
+
+// src/evals/metric.ts
+function defineMetric(metric) {
+  return metric;
+}
+
+// src/evals/metrics.ts
+import { z as z2 } from "zod";
+function exactMatch(options = {}) {
+  return {
+    name: options.name ?? "exact_match",
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActual(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (expected === void 0) {
+        return EvalOutcome.invalid("No expected value provided for exact match.");
+      }
+      const passed = stableComparable(actual) === stableComparable(expected);
+      return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });
+    }
+  };
+}
+function contains(options = {}) {
+  return {
+    name: options.name ?? "contains",
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (expected === void 0) {
+        return EvalOutcome.invalid("No expected value provided for contains.");
+      }
+      if (typeof expected !== "string" && !(expected instanceof RegExp)) {
+        return EvalOutcome.invalid("Contains expected value must be a string or RegExp.");
+      }
+      const passed = expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);
+      return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });
+    }
+  };
+}
+function notContains(options = {}) {
+  return {
+    name: options.name ?? "not_contains",
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (expected === void 0) {
+        return EvalOutcome.invalid("No expected value provided for notContains.");
+      }
+      if (typeof expected !== "string" && !(expected instanceof RegExp)) {
+        return EvalOutcome.invalid("notContains expected value must be a string or RegExp.");
+      }
+      const found = textExpectationMatches(expected, actual);
+      return found ? EvalOutcome.fail(false, {
+        comment: `Output contained forbidden value ${String(expected)}.`
+      }) : EvalOutcome.pass(true);
+    }
+  };
+}
+function containsAll(options) {
+  return containsListMetric("contains_all", "all", options);
+}
+function containsAny(options) {
+  return containsListMetric("contains_any", "any", options);
+}
+function containsListMetric(defaultName, mode, options) {
+  return {
+    name: options.name ?? defaultName,
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (!Array.isArray(expected) || expected.length === 0) {
+        return EvalOutcome.invalid(`${defaultName} expected value must be a non-empty array.`);
+      }
+      if (expected.some((value) => typeof value !== "string" && !(value instanceof RegExp))) {
+        return EvalOutcome.invalid(`${defaultName} expected values must be strings or RegExp.`);
+      }
+      const matches2 = expected.map((value) => textExpectationMatches(value, actual));
+      const passed = mode === "all" ? matches2.every(Boolean) : matches2.some(Boolean);
+      if (passed) return EvalOutcome.pass(true);
+      const missing = expected.filter((_, index) => !matches2[index]).map(String);
+      const comment = mode === "all" ? `Output was missing: ${missing.join(", ")}.` : `Output matched none of: ${expected.map(String).join(", ")}.`;
+      return EvalOutcome.fail(false, { comment });
+    }
+  };
+}
+function matches(options) {
+  return regexMetric("matches", false, options);
+}
+function doesNotMatch(options) {
+  return regexMetric("does_not_match", true, options);
+}
+function regexMetric(defaultName, negate, options) {
+  return {
+    name: options.name ?? defaultName,
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (!(expected instanceof RegExp)) {
+        return EvalOutcome.invalid(`${defaultName} expected value must be a RegExp.`);
+      }
+      const matched = regexMatches(expected, actual);
+      const passed = negate ? !matched : matched;
+      return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, {
+        comment: negate ? `Output matched forbidden pattern ${String(expected)}.` : `Output did not match ${String(expected)}.`
+      });
+    }
+  };
+}
+function maxLength(options) {
+  return {
+    name: options.name ?? "max_length",
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const max = await resolveOption(options.max, args);
+      if (!Number.isInteger(max) || max < 0) {
+        return EvalOutcome.invalid("maxLength max must be a non-negative integer.");
+      }
+      const length = Array.from(actual).length;
+      return length <= max ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Output length ${length} exceeded maximum ${max}.` });
+    }
+  };
+}
+function requiredFields(options) {
+  return {
+    name: options.name ?? "required_fields",
+    required: options.required ?? true,
+    dataType: "BOOLEAN",
+    direction: "higher_is_better",
+    threshold: 1,
+    async evaluate(args) {
+      const actual = await resolveActual(options.actual, args);
+      const expected = await resolveOption(options.expected, args);
+      if (!Array.isArray(expected) || expected.length === 0 || expected.some((field) => typeof field !== "string" || field.length === 0)) {
+        return EvalOutcome.invalid(
+          "requiredFields expected value must be a non-empty string array."
+        );
+      }
+      if (typeof actual !== "object" || actual === null || Array.isArray(actual)) {
+        return EvalOutcome.invalid("requiredFields actual value must be an object.");
+      }
+      const missing = expected.filter((field) => !Object.hasOwn(actual, field));
+      return missing.length === 0 ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Missing required fields: ${missing.join(", ")}.` });
+    }
+  };
+}
+function regexMatches(pattern, text) {
+  pattern.lastIndex = 0;
+  const matched = pattern.test(text);
+  pattern.lastIndex = 0;
+  return matched;
+}
+function textExpectationMatches(expected, actual) {
+  return expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);
+}
+async function resolveOption(value, args) {
+  return typeof value === "function" ? value(args) : value;
+}
+function semanticSimilarity(options) {
+  return {
+    name: options.name ?? "semantic_similarity",
+    required: options.required ?? true,
+    dataType: "NUMERIC",
+    direction: "higher_is_better",
+    threshold: options.threshold,
+    async evaluate(args) {
+      const actual = await resolveActualText(options.actual, args);
+      const expected = await resolveExpected(options.expected, args);
+      if (expected === void 0) {
+        return EvalOutcome.invalid("No expected value provided for semantic similarity.");
+      }
+      if (typeof expected !== "string") {
+        return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
+      }
+      const [actualEmbedding, expectedEmbedding] = await Promise.all([
+        embedText(options.model, actual),
+        embedText(options.model, expected)
+      ]);
+      const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
+      return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
+    }
+  };
+}
+function llmJudge(options) {
+  const extractor = new ExtractorBuilder(options.model, options.schema).instructions(
+    options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema."
+  ).retries(options.retries ?? 0).build();
+  return {
+    name: options.name ?? "llm_judge",
+    required: options.required ?? true,
+    async evaluate(args) {
+      try {
+        const result = await extractor.extractWithUsage(
+          await resolveJudgePrompt(options.prompt, args)
+        );
+        return options.passes(result.data) ? EvalOutcome.pass(result.data, { usage: result.usage }) : EvalOutcome.fail(result.data, { usage: result.usage });
+      } catch (error) {
+        return EvalOutcome.invalid(errorMessage(error));
+      }
+    }
+  };
+}
+function llmScore(options) {
+  const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
+  const extractor = new ExtractorBuilder(
+    options.model,
+    z2.object({
+      score: z2.number(),
+      feedback: z2.string()
+    })
+  ).instructions(
+    options.instructions ?? `Score the eval case against these criteria:
+${criteria}
+
+Return a score between 0 and 1 and brief feedback.`
+  ).retries(options.retries ?? 0).build();
+  return {
+    name: options.name ?? "llm_score",
+    required: options.required ?? true,
+    dataType: "NUMERIC",
+    projectScore: (score) => score.score,
+    direction: "higher_is_better",
+    threshold: options.threshold,
+    async evaluate(args) {
+      try {
+        const result = await extractor.extractWithUsage(
+          await resolveJudgePrompt(options.prompt, args)
+        );
+        const score = result.data;
+        if (score.score < 0 || score.score > 1) {
+          return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
+            score,
+            usage: result.usage
+          });
+        }
+        return score.score >= options.threshold ? EvalOutcome.pass(score, { comment: score.feedback, usage: result.usage }) : EvalOutcome.fail(score, { comment: score.feedback, usage: result.usage });
+      } catch (error) {
+        return EvalOutcome.invalid(errorMessage(error));
+      }
+    }
+  };
+}
+
+// src/evals/suite.ts
+function defineEvalCases(cases) {
+  return cases;
+}
+function defineEvalSuite(options) {
+  if (options !== void 0) return options;
+  return {
+    defineMetric(metric) {
+      return metric;
+    }
+  };
 }
 export {
+  EvalAssertionError,
   EvalOutcome,
+  abstention,
   agentEvalTarget,
   answerRelevancy,
+  assertEvalOutcomes,
+  assertEvalTotals,
   contains,
+  containsAll,
+  containsAny,
   defaultEvalTraceSelector,
+  defineEvalCases,
+  defineEvalSuite,
   defineMetric,
+  doesNotMatch,
+  evalExitCode,
   exactMatch,
   faithfulness,
   gEval,
@@ -1464,10 +2102,17 @@ export {
   knowledgeRetention,
   llmJudge,
   llmScore,
+  matches,
+  maxLength,
+  notContains,
+  printEvalResult,
   projectEvalOutcome,
   promptAlignment,
+  requiredFields,
   resolveEvalTraceRef,
+  runEvalCli,
   runEvalSuite,
+  selectPromptOutput,
   semanticSimilarity,
   summarization,
   turnRelevancy