@anvia/core 0.22.0 → 0.24.0

package/dist/evals/index.js

@@ -1,20 +1,16 @@
 import {
   ExtractorBuilder
-} from "../chunk-KYEBWN3E.js";
-import "../chunk-4HWN7734.js";
-import "../chunk-YEBIVZM2.js";
-import "../chunk-VXONXP2G.js";
+} from "../chunk-YLMDBQO4.js";
+import "../chunk-JI5YZWNT.js";
+import "../chunk-DALYP4PX.js";
+import "../chunk-BALBBVI3.js";
 import "../chunk-YK4WAAS4.js";
 import "../chunk-XUUY2L2D.js";
-import "../chunk-2NDQHV7V.js";
-import "../chunk-4BGN6PYF.js";
+import "../chunk-HYUYZEAJ.js";
+import "../chunk-MELYDKWQ.js";
 import "../chunk-2ODTMRHP.js";
 import "../chunk-UQX6EXWG.js";
 import "../chunk-MRQLEK2B.js";
-import "../chunk-Q25IWLBT.js";
-import "../chunk-MC3CCKEB.js";
-import "../chunk-ADH7NNCS.js";
-import "../chunk-WQKHFADH.js";
 import {
   cosineSimilarity,
   embedText
@@ -22,23 +18,15 @@ import {
 import {
   mapWithConcurrency
 } from "../chunk-OIMLU4SF.js";
+import "../chunk-Q25IWLBT.js";
+import "../chunk-MC3CCKEB.js";
+import {
+  Usage
+} from "../chunk-ADH7NNCS.js";
+import "../chunk-WQKHFADH.js";
 import "../chunk-CWUJUSOS.js";
 
-// src/evals/agent-target.ts
-function agentEvalTarget(agent, options = {}) {
-  return async (input, testCase) => {
-    const prompt = options.prompt?.(input, testCase) ?? String(input);
-    const response = await agent.prompt(prompt).send();
-    return options.output === void 0 ? response : options.output(response, testCase);
-  };
-}
-
-// src/evals/metric.ts
-function defineMetric(metric) {
-  return metric;
-}
-
-// src/evals/metrics.ts
+// src/evals/advanced-metrics.ts
 import { z } from "zod";
 
 // src/evals/format.ts
@@ -68,6 +56,35 @@ function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
+// src/evals/judge.ts
+async function runJudge(args) {
+  const result = await new ExtractorBuilder(args.model, args.schema).instructions(args.instructions).temperature(0).retries(args.retries).build().extractWithUsage(args.prompt);
+  return { data: result.data, usage: result.usage };
+}
+function addUsage(...values) {
+  return values.reduce((total, usage) => Usage.add(total, usage), Usage.empty());
+}
+function evaluationMetadata(details, usage) {
+  const evaluation = {
+    ...details,
+    usage: usageToJson(usage)
+  };
+  return { evaluation };
+}
+function usageToJson(usage) {
+  const value = {
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    totalTokens: usage.totalTokens,
+    cachedInputTokens: usage.cachedInputTokens,
+    cacheCreationInputTokens: usage.cacheCreationInputTokens
+  };
+  if (usage.details !== void 0) {
+    value.details = { ...usage.details };
+  }
+  return value;
+}
+
 // src/evals/outcome.ts
 var EvalOutcome = {
   pass(score, options = {}) {
@@ -145,7 +162,872 @@ async function resolveJudgePrompt(selector, args) {
   ].join("\n\n");
 }
 
+// src/evals/advanced-metrics.ts
+var statementsSchema = z.object({ statements: z.array(z.string()) });
+var factsSchema = z.object({ facts: z.array(z.string()) });
+var questionsSchema = z.object({ questions: z.array(z.string()) });
+var answersSchema = z.object({ answers: z.array(z.enum(["yes", "no"])) });
+var verdictsSchema = z.object({
+  verdicts: z.array(
+    z.object({
+      verdict: z.enum(["yes", "no", "idk"]),
+      reason: z.string().optional()
+    })
+  )
+});
+var binaryVerdictsSchema = z.object({
+  verdicts: z.array(
+    z.object({
+      verdict: z.enum(["yes", "no"]),
+      reason: z.string()
+    })
+  )
+});
+var binaryVerdictSchema = z.object({
+  verdict: z.enum(["yes", "no"]),
+  reason: z.string()
+});
+var reasonSchema = z.object({ reason: z.string() });
+function answerRelevancy(options) {
+  const config = metricConfig(options, "answer_relevancy");
+  return numericMetric(config.name, async (args) => {
+    try {
+      const input = await resolveInput(options.input, args);
+      const actual = await resolveActualText(options.actual, args);
+      const statementResult = await runJudge({
+        model: options.model,
+        schema: statementsSchema,
+        instructions: "Break the answer into concise, independently assessable statements. Return every substantive statement using the schema.",
+        prompt: `Answer:
+${actual}`,
+        retries: config.retries
+      });
+      const statements = statementResult.data.statements;
+      let verdicts = [];
+      let usage = statementResult.usage;
+      if (statements.length > 0) {
+        const verdictResult = await runJudge({
+          model: options.model,
+          schema: verdictsSchema,
+          instructions: "Classify each answer statement for relevance to the user input. Use yes for relevant, no for irrelevant, and idk only when relevance is genuinely indeterminate. Preserve order and return one verdict per statement.",
+          prompt: jsonPrompt({ input, statements }),
+          retries: config.retries
+        });
+        verdicts = verdictResult.data.verdicts;
+        assertSameLength("answer relevancy verdicts", statements, verdicts);
+        usage = addUsage(usage, verdictResult.usage);
+      }
+      const score = verdicts.length === 0 ? 1 : verdicts.filter((verdict) => verdict.verdict !== "no").length / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "answer relevancy",
+        score,
+        evidence: { input, verdicts }
+      });
+      usage = addUsage(usage, reasonResult.usage);
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { statements, verdicts },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function promptAlignment(options) {
+  if (options.promptInstructions.length === 0) {
+    throw new TypeError("promptAlignment requires at least one prompt instruction.");
+  }
+  const config = metricConfig(options, "prompt_alignment");
+  return numericMetric(config.name, async (args) => {
+    try {
+      const input = await resolveInput(options.input, args);
+      const actual = await resolveActualText(options.actual, args);
+      const verdictResult = await runJudge({
+        model: options.model,
+        schema: binaryVerdictsSchema,
+        instructions: "Determine whether the answer follows each prompt instruction. Preserve order and return exactly one yes or no verdict per instruction.",
+        prompt: jsonPrompt({ input, actual, instructions: options.promptInstructions }),
+        retries: config.retries
+      });
+      const verdicts = verdictResult.data.verdicts;
+      assertSameLength("prompt alignment verdicts", options.promptInstructions, verdicts);
+      const score = verdicts.filter((verdict) => verdict.verdict === "yes").length / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "prompt alignment",
+        score,
+        evidence: { verdicts }
+      });
+      const usage = addUsage(verdictResult.usage, reasonResult.usage);
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { promptInstructions: options.promptInstructions, verdicts },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function jsonCorrectness(options) {
+  const threshold = validateThreshold(options.threshold ?? 0.5);
+  const retries = validateRetries(options.retries ?? 0);
+  const includeReason = options.includeReason ?? true;
+  return numericMetric(options.name ?? "json_correctness", async (args) => {
+    try {
+      const actual = await resolveActualText(options.actual, args);
+      let parsed;
+      let validationError;
+      try {
+        parsed = JSON.parse(actual);
+        const result = options.schema.safeParse(parsed);
+        if (!result.success) {
+          validationError = z.prettifyError(result.error);
+        }
+      } catch (error) {
+        validationError = errorMessage(error);
+      }
+      const score = validationError === void 0 ? 1 : 0;
+      let comment;
+      let usage = Usage.empty();
+      if (includeReason) {
+        if (score === 1) {
+          comment = "The generated JSON is syntactically valid and matches the expected schema.";
+        } else if (options.model === void 0) {
+          comment = validationError;
+        } else {
+          const reasonResult = await runJudge({
+            model: options.model,
+            schema: reasonSchema,
+            instructions: "Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.",
+            prompt: jsonPrompt({ actual, validationError }),
+            retries
+          });
+          comment = reasonResult.data.reason;
+          usage = reasonResult.usage;
+        }
+      }
+      return higherOutcome({
+        score,
+        threshold,
+        strictMode: options.strictMode ?? true,
+        comment,
+        details: validationError === void 0 ? {} : { validationError },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function hallucination(options) {
+  const config = metricConfig(options, "hallucination");
+  return numericMetric(config.name, async (args) => {
+    try {
+      const actual = await resolveActualText(options.actual, args);
+      const context = await resolveStringList(options.context, args.case.context, args, "context");
+      const verdictResult = await runJudge({
+        model: options.model,
+        schema: binaryVerdictsSchema,
+        instructions: "Compare the answer with each trusted context. Use yes when the answer is factually aligned with that context and no when it contradicts it. Preserve order and return one verdict per context.",
+        prompt: jsonPrompt({ actual, context }),
+        retries: config.retries
+      });
+      const verdicts = verdictResult.data.verdicts;
+      assertSameLength("hallucination verdicts", context, verdicts);
+      const score = verdicts.filter((verdict) => verdict.verdict === "no").length / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "hallucination",
+        score,
+        evidence: { verdicts }
+      });
+      const usage = addUsage(verdictResult.usage, reasonResult.usage);
+      return lowerOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { verdicts },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function faithfulness(options) {
+  const config = metricConfig(options, "faithfulness");
+  const truthsExtractionLimit = validateOptionalNonNegativeInteger(
+    options.truthsExtractionLimit,
+    "truthsExtractionLimit"
+  );
+  return numericMetric(config.name, async (args) => {
+    try {
+      const actual = await resolveActualText(options.actual, args);
+      const retrievalContext = await resolveStringList(
+        options.retrievalContext,
+        args.case.retrievalContext,
+        args,
+        "retrievalContext"
+      );
+      const [truthResult, claimResult] = await Promise.all([
+        runJudge({
+          model: options.model,
+          schema: factsSchema,
+          instructions: truthsExtractionInstructions(truthsExtractionLimit),
+          prompt: jsonPrompt({ retrievalContext }),
+          retries: config.retries
+        }),
+        runJudge({
+          model: options.model,
+          schema: factsSchema,
+          instructions: "Extract every concise factual claim made by the answer. Return claims in the facts array and omit opinions or purely stylistic text.",
+          prompt: `Answer:
+${actual}`,
+          retries: config.retries
+        })
+      ]);
+      const truths = limitValues(truthResult.data.facts, truthsExtractionLimit);
+      const claims = claimResult.data.facts;
+      let verdicts = [];
+      let usage = addUsage(truthResult.usage, claimResult.usage);
+      if (claims.length > 0) {
+        const verdictResult = await runJudge({
+          model: options.model,
+          schema: verdictsSchema,
+          instructions: "Determine whether each answer claim is supported by the supplied truths. Use yes for supported, no for contradicted or unsupported, and idk for genuinely ambiguous support. Preserve order and return one verdict per claim.",
+          prompt: jsonPrompt({ truths, claims }),
+          retries: config.retries
+        });
+        verdicts = verdictResult.data.verdicts;
+        assertSameLength("faithfulness verdicts", claims, verdicts);
+        usage = addUsage(usage, verdictResult.usage);
+      }
+      const penalizeAmbiguousClaims = options.penalizeAmbiguousClaims ?? false;
+      const supported = verdicts.filter(
+        (verdict) => verdict.verdict === "yes" || verdict.verdict === "idk" && !penalizeAmbiguousClaims
+      ).length;
+      const score = verdicts.length === 0 ? 1 : supported / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "faithfulness",
+        score,
+        evidence: { verdicts, penalizeAmbiguousClaims }
+      });
+      usage = addUsage(usage, reasonResult.usage);
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { truths, claims, verdicts, penalizeAmbiguousClaims },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function summarization(options) {
+  const config = metricConfig(options, "summarization");
+  const questionCount = validatePositiveInteger(options.questionCount ?? 5, "questionCount");
+  const truthsExtractionLimit = validateOptionalNonNegativeInteger(
+    options.truthsExtractionLimit,
+    "truthsExtractionLimit"
+  );
+  const suppliedQuestions = options.assessmentQuestions !== void 0 && options.assessmentQuestions.length > 0 ? [...options.assessmentQuestions] : void 0;
+  return numericMetric(config.name, async (args) => {
+    try {
+      const input = await resolveInput(options.input, args);
+      const actual = await resolveActualText(options.actual, args);
+      const questionPromise = suppliedQuestions === void 0 ? runJudge({
+        model: options.model,
+        schema: questionsSchema,
+        instructions: `Generate exactly ${questionCount} important yes-or-no assessment questions whose answers capture the source text's essential information.`,
+        prompt: `Source text:
+${input}`,
+        retries: config.retries
+      }) : Promise.resolve({
+        data: { questions: suppliedQuestions },
+        usage: Usage.empty()
+      });
+      const [truthResult, claimResult, questionResult] = await Promise.all([
+        runJudge({
+          model: options.model,
+          schema: factsSchema,
+          instructions: truthsExtractionInstructions(truthsExtractionLimit),
+          prompt: `Source text:
+${input}`,
+          retries: config.retries
+        }),
+        runJudge({
+          model: options.model,
+          schema: factsSchema,
+          instructions: "Extract every concise factual claim made by the summary. Return claims in the facts array.",
+          prompt: `Summary:
+${actual}`,
+          retries: config.retries
+        }),
+        questionPromise
+      ]);
+      const truths = limitValues(truthResult.data.facts, truthsExtractionLimit);
+      const claims = claimResult.data.facts;
+      const questions = questionResult.data.questions;
+      if (questions.length === 0) {
+        throw new Error("Summarization assessment questions must not be empty.");
+      }
+      const sourceAnswerPromise = runJudge({
+        model: options.model,
+        schema: answersSchema,
+        instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
+        prompt: jsonPrompt({ questions, text: input }),
+        retries: config.retries
+      });
+      const summaryAnswerPromise = runJudge({
+        model: options.model,
+        schema: answersSchema,
+        instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
+        prompt: jsonPrompt({ questions, text: actual }),
+        retries: config.retries
+      });
+      const alignmentPromise = claims.length === 0 ? Promise.resolve({ data: { verdicts: [] }, usage: Usage.empty() }) : runJudge({
+        model: options.model,
+        schema: verdictsSchema,
+        instructions: "Determine whether each summary claim is supported by the source truths. Use yes for supported, no for contradicted, and idk for unsupported filler or ambiguity. Preserve order.",
+        prompt: jsonPrompt({ truths, claims }),
+        retries: config.retries
+      });
+      const [sourceAnswerResult, summaryAnswerResult, alignmentResult] = await Promise.all([
+        sourceAnswerPromise,
+        summaryAnswerPromise,
+        alignmentPromise
+      ]);
+      assertSameLength("source assessment answers", questions, sourceAnswerResult.data.answers);
+      assertSameLength("summary assessment answers", questions, summaryAnswerResult.data.answers);
+      assertSameLength("summarization alignment verdicts", claims, alignmentResult.data.verdicts);
+      const alignmentVerdicts = alignmentResult.data.verdicts;
+      const alignmentScore = alignmentVerdicts.length === 0 ? 0 : alignmentVerdicts.filter((verdict) => verdict.verdict === "yes").length / alignmentVerdicts.length;
+      let coverageTotal = 0;
+      let coverageMatched = 0;
+      const coverageVerdicts = questions.map((question, index) => {
+        const originalVerdict = sourceAnswerResult.data.answers[index];
+        const summaryVerdict = summaryAnswerResult.data.answers[index];
+        if (originalVerdict === "yes") {
+          coverageTotal += 1;
+          if (summaryVerdict === "yes") coverageMatched += 1;
+        }
+        return { question, originalVerdict, summaryVerdict };
+      });
+      const coverageScore = coverageTotal === 0 ? 0 : coverageMatched / coverageTotal;
+      const score = Math.min(alignmentScore, coverageScore);
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "summarization",
+        score,
+        evidence: { alignmentVerdicts, coverageVerdicts, alignmentScore, coverageScore }
+      });
+      const usage = addUsage(
+        truthResult.usage,
+        claimResult.usage,
+        questionResult.usage,
+        sourceAnswerResult.usage,
+        summaryAnswerResult.usage,
+        alignmentResult.usage,
+        reasonResult.usage
+      );
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: {
+          truths,
+          claims,
+          assessmentQuestions: questions,
+          alignmentVerdicts,
+          coverageVerdicts,
+          scoreBreakdown: { alignment: alignmentScore, coverage: coverageScore }
+        },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function gEval(options) {
+  if (options.name.trim().length === 0) throw new TypeError("gEval name must not be empty.");
+  if (options.evaluationParams.length === 0) {
+    throw new TypeError("gEval requires at least one evaluation parameter.");
+  }
+  if (options.criteria === void 0 === (options.evaluationSteps === void 0)) {
+    throw new TypeError("gEval requires exactly one of criteria or evaluationSteps.");
+  }
+  if (options.criteria !== void 0 && options.criteria.trim().length === 0) {
+    throw new TypeError("gEval criteria must not be empty.");
+  }
+  if (options.evaluationSteps !== void 0 && options.evaluationSteps.length === 0) {
+    throw new TypeError("gEval evaluationSteps must not be empty.");
+  }
+  const config = metricConfig(options, options.name);
+  const rubric = validateRubric(options.rubric);
+  const scoreRange = rubric.length === 0 ? [0, 10] : [rubric[0]?.scoreRange[0] ?? 0, rubric.at(-1)?.scoreRange[1] ?? 10];
+  let generatedStepsPromise;
+  let generatedUsageClaimed = false;
+  async function resolveSteps() {
+    if (options.evaluationSteps !== void 0) {
+      return { steps: options.evaluationSteps, usage: Usage.empty() };
+    }
+    if (generatedStepsPromise === void 0) {
+      generatedStepsPromise = runJudge({
+        model: options.model,
+        schema: z.object({ steps: z.array(z.string()) }),
+        instructions: "Generate three or four concise evaluation steps from the criteria. Explain how the selected parameters should be judged in relation to one another.",
+        prompt: jsonPrompt({ criteria: options.criteria, parameters: options.evaluationParams }),
+        retries: config.retries
+      }).catch((error) => {
+        generatedStepsPromise = void 0;
+        throw error;
+      });
+    }
+    const result = await generatedStepsPromise;
+    if (result.data.steps.length === 0) throw new Error("G-Eval generated no evaluation steps.");
+    const usage = generatedUsageClaimed ? Usage.empty() : result.usage;
+    generatedUsageClaimed = true;
+    return { steps: result.data.steps, usage };
+  }
+  return numericMetric(config.name, async (args) => {
+    try {
+      const parameters = await resolveGEvalParameters(options, args);
+      const stepsResult = await resolveSteps();
+      const scoreResult = await runJudge({
+        model: options.model,
+        schema: z.object({ score: z.number(), reason: z.string() }),
+        instructions: config.strictMode ? "Apply the evaluation steps and return score 1 only for complete compliance, otherwise 0. Give a concise evidence-based reason." : `Apply the evaluation steps and return an integer score from ${scoreRange[0]} through ${scoreRange[1]}, plus a concise evidence-based reason.`,
+        prompt: jsonPrompt({
+          evaluationSteps: stepsResult.steps,
+          rubric,
+          parameters
+        }),
+        retries: config.retries
+      });
+      const rawScore = scoreResult.data.score;
+      if (!Number.isFinite(rawScore) || config.strictMode && rawScore !== 0 && rawScore !== 1 || !config.strictMode && (rawScore < scoreRange[0] || rawScore > scoreRange[1])) {
+        throw new RangeError(`G-Eval score ${rawScore} is outside the requested range.`);
+      }
+      const score = config.strictMode ? rawScore : (rawScore - scoreRange[0]) / (scoreRange[1] - scoreRange[0]);
+      const usage = addUsage(stepsResult.usage, scoreResult.usage);
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: config.includeReason ? scoreResult.data.reason : void 0,
+        details: {
+          evaluationSteps: stepsResult.steps,
+          evaluationParams: options.evaluationParams,
+          rawScore,
+          scoreRange: [scoreRange[0], scoreRange[1]],
+          rubric: rubric.map((entry) => ({
+            scoreRange: [entry.scoreRange[0], entry.scoreRange[1]],
+            expectedOutcome: entry.expectedOutcome
+          }))
+        },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function turnRelevancy(options) {
+  const config = metricConfig(options, "turn_relevancy");
+  const windowSize = validatePositiveInteger(options.windowSize ?? 10, "windowSize");
+  const concurrency = validatePositiveInteger(options.concurrency ?? 4, "concurrency");
+  return numericMetric(config.name, async (args) => {
+    try {
+      const turns = await resolveTurns(options.turns, args);
+      const interactions = unitInteractions(turns);
+      const windows = interactions.map(
+        (_, index) => interactions.slice(Math.max(0, index - windowSize + 1), index + 1).flat()
+      );
+      const verdictResults = await mapWithConcurrency(
+        windows,
+        concurrency,
+        (window) => runJudge({
+          model: options.model,
+          schema: binaryVerdictSchema,
+          instructions: "Judge whether the final assistant reply is relevant to the preceding conversation. Return yes for relevant and no for irrelevant, with a concise reason.",
+          prompt: jsonPrompt({ turns: window }),
+          retries: config.retries
+        })
+      );
+      const verdicts = verdictResults.map((result, index) => ({
+        interaction: index + 1,
+        ...result.data
+      }));
+      const score = verdicts.length === 0 ? 1 : verdicts.filter((verdict) => verdict.verdict === "yes").length / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "turn relevancy",
+        score,
+        evidence: { verdicts }
+      });
+      const usage = addUsage(...verdictResults.map((result) => result.usage), reasonResult.usage);
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { windowSize, concurrency, interactionCount: interactions.length, verdicts },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function knowledgeRetention(options) {
+  const config = metricConfig(options, "knowledge_retention");
+  const concurrency = validatePositiveInteger(options.concurrency ?? 4, "concurrency");
+  return numericMetric(config.name, async (args) => {
+    try {
+      const turns = await resolveTurns(options.turns, args);
+      const userTurns = turns.map((turn, index) => ({ turn, index })).filter((entry) => entry.turn.role === "user");
+      const knowledgeResults = await mapWithConcurrency(
+        userTurns,
+        concurrency,
+        (entry) => runJudge({
+          model: options.model,
+          schema: factsSchema,
+          instructions: "Extract durable factual information newly supplied by the final user message. Use prior turns only to resolve references. Return concise facts; return an empty array when nothing new was supplied.",
+          prompt: jsonPrompt({
+            previousTurns: turns.slice(0, entry.index),
+            userMessage: entry.turn.content
+          }),
+          retries: config.retries
+        })
+      );
+      const knowledge = userTurns.map((entry, index) => ({
+        turnIndex: entry.index,
+        facts: knowledgeResults[index]?.data.facts ?? []
+      }));
+      const assistantChecks = turns.map((turn, index) => ({ turn, index })).filter((entry) => entry.turn.role === "assistant").map((entry) => ({
+        ...entry,
+        facts: knowledge.filter((item) => item.turnIndex < entry.index).flatMap((item) => item.facts)
+      })).filter((entry) => entry.facts.length > 0);
+      const verdictResults = await mapWithConcurrency(
+        assistantChecks,
+        concurrency,
+        (entry) => runJudge({
+          model: options.model,
+          schema: z.object({ attrition: z.boolean(), reason: z.string() }),
+          instructions: "Determine whether the assistant reply forgets, contradicts, or unnecessarily asks again for information already supplied by the user. Set attrition true only when knowledge was lost.",
+          prompt: jsonPrompt({ knownFacts: entry.facts, assistantReply: entry.turn.content }),
+          retries: config.retries
+        })
+      );
+      const verdicts = assistantChecks.map((entry, index) => ({
+        turnIndex: entry.index,
+        attrition: verdictResults[index]?.data.attrition ?? true,
+        reason: verdictResults[index]?.data.reason ?? "Missing knowledge-retention verdict."
+      }));
+      const score = verdicts.length === 0 ? 1 : verdicts.filter((verdict) => !verdict.attrition).length / verdicts.length;
+      const reasonResult = await maybeReason({
+        model: options.model,
+        includeReason: config.includeReason,
+        retries: config.retries,
+        metric: "knowledge retention",
+        score,
+        evidence: { verdicts }
+      });
+      const usage = addUsage(
+        ...knowledgeResults.map((result) => result.usage),
+        ...verdictResults.map((result) => result.usage),
+        reasonResult.usage
+      );
+      return higherOutcome({
+        score,
+        threshold: config.threshold,
+        strictMode: config.strictMode,
+        comment: reasonResult.reason,
+        details: { concurrency, knowledge, verdicts },
+        usage
+      });
+    } catch (error) {
+      return EvalOutcome.invalid(errorMessage(error));
+    }
+  });
+}
+function numericMetric(name, evaluate) {
+  return { name, dataType: "NUMERIC", evaluate };
+}
+function metricConfig(options, defaultName) {
+  return {
+    name: options.name ?? defaultName,
+    threshold: validateThreshold(options.threshold ?? 0.5),
+    strictMode: options.strictMode ?? false,
+    includeReason: options.includeReason ?? true,
+    retries: validateRetries(options.retries ?? 0)
+  };
+}
+function higherOutcome(args) {
+  const score = args.strictMode ? args.score === 1 ? 1 : 0 : args.score;
+  const threshold = args.strictMode ? 1 : args.threshold;
+  const options = {
+    comment: args.comment,
+    metadata: evaluationMetadata(
+      {
+        ...args.details,
+        scoreDirection: "higher_is_better",
+        threshold,
+        strictMode: args.strictMode
+      },
+      args.usage
+    )
+  };
+  return score >= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);
+}
+function lowerOutcome(args) {
+  const score = args.strictMode ? args.score === 0 ? 0 : 1 : args.score;
+  const threshold = args.strictMode ? 0 : args.threshold;
+  const options = {
+    comment: args.comment,
+    metadata: evaluationMetadata(
+      {
+        ...args.details,
+        scoreDirection: "lower_is_better",
+        threshold,
+        strictMode: args.strictMode
+      },
+      args.usage
+    )
+  };
+  return score <= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);
+}
+async function maybeReason(args) {
+  if (!args.includeReason) return { usage: Usage.empty() };
+  const result = await runJudge({
+    model: args.model,
+    schema: reasonSchema,
+    instructions: `Write a concise final explanation for the ${args.metric} score. Ground it only in the supplied evidence and do not repeat the numeric score.`,
+    prompt: jsonPrompt({ score: args.score, evidence: args.evidence }),
+    retries: args.retries
+  });
+  return { reason: result.data.reason, usage: result.usage };
+}
+async function resolveInput(selector, args) {
+  return selector === void 0 ? formatValue(args.case.input) : selector(args);
+}
+async function resolveStringList(selectorOrValue, fallback, args, label) {
+  const value = selectorOrValue === void 0 ? fallback : typeof selectorOrValue === "function" ? await selectorOrValue(args) : selectorOrValue;
+  if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string")) {
+    throw new TypeError(`${label} must be a non-empty array of strings.`);
+  }
+  return value;
+}
+async function resolveGEvalParameters(options, args) {
+  const parameters = {};
+  for (const parameter of options.evaluationParams) {
+    if (parameter === "input") parameters.input = await resolveInput(options.input, args);
+    if (parameter === "actualOutput") {
+      parameters.actualOutput = await resolveActualText(options.actual, args);
+    }
+    if (parameter === "expectedOutput") {
+      const expected = options.expected === void 0 ? args.case.expected : await options.expected(args);
+      if (expected === void 0) throw new Error("G-Eval expectedOutput is missing.");
+      parameters.expectedOutput = toJsonValue(expected);
+    }
+    if (parameter === "context") {
+      parameters.context = await resolveStringList(
+        options.context,
+        args.case.context,
+        args,
+        "context"
+      );
+    }
+    if (parameter === "retrievalContext") {
+      parameters.retrievalContext = await resolveStringList(
+        options.retrievalContext,
+        args.case.retrievalContext,
+        args,
+        "retrievalContext"
+      );
+    }
+    if (parameter === "metadata") parameters.metadata = args.case.metadata ?? {};
+  }
+  return parameters;
+}
+function validateRubric(rubric) {
+  if (rubric === void 0 || rubric.length === 0) return [];
+  const sorted = [...rubric].sort((left, right) => left.scoreRange[0] - right.scoreRange[0]);
+  for (const [index, entry] of sorted.entries()) {
+    const [start, end] = entry.scoreRange;
+    if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end > 10 || start > end) {
+      throw new RangeError(
+        "G-Eval rubric score ranges must be ordered integers from 0 through 10."
+      );
+    }
+    if (entry.expectedOutcome.trim().length === 0) {
+      throw new TypeError("G-Eval rubric expectedOutcome must not be empty.");
+    }
+    const next = sorted[index + 1];
+    if (next !== void 0 && end >= next.scoreRange[0]) {
+      throw new RangeError("G-Eval rubric score ranges must not overlap.");
+    }
+  }
+  const first = sorted[0];
+  const last = sorted.at(-1);
+  if (first !== void 0 && last !== void 0 && first.scoreRange[0] === last.scoreRange[1]) {
+    throw new RangeError("G-Eval rubric score range must span more than one value.");
+  }
+  return sorted;
+}
+function truthsExtractionInstructions(limit) {
+  return limit === void 0 ? "Extract concise, factual, undisputed truths from the supplied source material, ordered by importance. Return them in the facts array." : `Extract at most ${limit} concise, factual, undisputed truths from the supplied source material, ordered by importance. Return them in the facts array.`;
+}
+function limitValues(values, limit) {
+  return limit === void 0 ? values : values.slice(0, limit);
+}
+function validateThreshold(value) {
+  if (!Number.isFinite(value) || value < 0 || value > 1) {
+    throw new RangeError("Eval metric threshold must be between 0 and 1.");
+  }
+  return value;
+}
+function validateRetries(value) {
+  if (!Number.isInteger(value) || value < 0) {
+    throw new RangeError("Eval metric retries must be a non-negative integer.");
+  }
+  return value;
+}
+function validatePositiveInteger(value, label) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new RangeError(`${label} must be a positive integer.`);
+  }
+  return value;
+}
+function validateOptionalNonNegativeInteger(value, label) {
+  if (value === void 0) return void 0;
+  if (!Number.isInteger(value) || value < 0) {
+    throw new RangeError(`${label} must be a non-negative integer.`);
+  }
+  return value;
+}
+function assertSameLength(label, inputs, outputs) {
+  if (inputs.length !== outputs.length) {
+    throw new Error(`${label} count ${outputs.length} did not match input count ${inputs.length}.`);
+  }
+}
+function jsonPrompt(value) {
+  return JSON.stringify(value, null, 2);
+}
+function toJsonValue(value) {
+  try {
+    return JSON.parse(JSON.stringify(value));
+  } catch {
+    return String(value);
+  }
+}
+function normalizeEvalTurns(value) {
+  const source = conversationArray(value);
+  if (source === void 0) return void 0;
+  const turns = [];
+  for (const entry of source) {
+    if (typeof entry !== "object" || entry === null) continue;
+    const role = entry.role;
+    if (role !== "user" && role !== "assistant") continue;
+    const content = entry.content;
+    const text = contentText(content);
+    if (text.length === 0) continue;
+    const metadata = entry.metadata;
+    turns.push(
+      typeof metadata === "object" && metadata !== null && !Array.isArray(metadata) ? { role, content: text, metadata } : { role, content: text }
+    );
+  }
+  return turns.length === 0 ? void 0 : turns;
+}
+async function resolveTurns(selector, args) {
+  const source = selector === void 0 ? args.output : await selector(args);
+  const turns = normalizeEvalTurns(source);
+  if (turns === void 0) {
+    throw new TypeError(
+      "Conversational eval requires non-empty EvalTurn[], Message[], or an output with messages."
+    );
+  }
+  return turns;
+}
+function unitInteractions(turns) {
+  const interactions = [];
+  let current = [];
+  let hasUser = false;
+  for (const turn of turns) {
+    if (current.at(-1)?.role === "assistant" && turn.role === "user" && hasUser) {
+      interactions.push(current);
+      current = [turn];
+      hasUser = true;
+      continue;
+    }
+    current.push(turn);
+    if (turn.role === "user") hasUser = true;
+  }
+  if (current.length > 1 && current.at(-1)?.role === "assistant" && hasUser) {
+    interactions.push(current);
+  }
+  return interactions;
+}
+function conversationArray(value) {
+  if (Array.isArray(value)) return value;
+  if (typeof value === "object" && value !== null && "messages" in value) {
+    const messages = value.messages;
+    return Array.isArray(messages) ? messages : void 0;
+  }
+  return void 0;
+}
+function contentText(content) {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content.flatMap(
+    (part) => typeof part === "object" && part !== null && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string" ? [part.text] : []
+  ).join("\n");
+}
+
+// src/evals/agent-target.ts
+function agentEvalTarget(agent, options = {}) {
+  return async (input, testCase) => {
+    const prompt = options.prompt?.(input, testCase) ?? String(input);
+    const response = await agent.prompt(prompt).send();
+    return options.output === void 0 ? response : options.output(response, testCase);
+  };
+}
+
+// src/evals/metric.ts
+function defineMetric(metric) {
+  return metric;
+}
+
 // src/evals/metrics.ts
+import { z as z2 } from "zod";
 function exactMatch(options = {}) {
   return {
     name: options.name ?? "exact_match",
@@ -224,9 +1106,9 @@ function llmScore(options) {
   const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
   const extractor = new ExtractorBuilder(
     options.model,
-    z.object({
-      score: z.number(),
-      feedback: z.string()
+    z2.object({
+      score: z2.number(),
+      feedback: z2.string()
     })
   ).instructions(
     options.instructions ?? `Score the eval case against these criteria:
@@ -252,23 +1134,174 @@ Return a score between 0 and 1 and brief feedback.`
   };
 }
 
+// src/evals/reporting.ts
+function projectEvalOutcome(outcome, dataType) {
+  const value = projectScoreValue(outcome, dataType);
+  const projection = {
+    outcome: outcome.outcome,
+    value,
+    label: typeof value === "string" ? value : outcome.outcome
+  };
+  if (typeof value === "number") projection.numericValue = value;
+  if (typeof value === "string") projection.categoricalValue = value;
+  const explanation = outcome.comment ?? (outcome.outcome === "invalid" ? outcome.reason : void 0);
+  if (explanation !== void 0) projection.explanation = explanation;
+  return projection;
+}
+function resolveEvalTraceRef(args) {
+  return traceFromCarrier(args.output) ?? traceFromCarrier(args.input) ?? traceFromMetadata(args.metadata);
+}
+function defaultEvalTraceSelector(args) {
+  return resolveEvalTraceRef({
+    output: args.output,
+    input: args.case.input,
+    metadata: args.case.metadata
+  });
+}
+function projectScoreValue(outcome, dataType) {
+  const score = outcome.score;
+  if (dataType === "CATEGORICAL") {
+    if (typeof score === "string") return score;
+    if (typeof score === "number") return String(score);
+    if (typeof score === "boolean") return score ? "true" : "false";
+    if (score === null || score === void 0) return outcome.outcome;
+    try {
+      return JSON.stringify(score) ?? outcome.outcome;
+    } catch {
+      return outcome.outcome;
+    }
+  }
+  if (dataType === "BOOLEAN") {
+    if (typeof score === "boolean") return score ? 1 : 0;
+    if (typeof score === "number") return score === 0 ? 0 : 1;
+    return outcome.outcome === "pass" ? 1 : 0;
+  }
+  if (typeof score === "number") return score;
+  if (typeof score === "boolean") return score ? 1 : 0;
+  if (typeof score === "object" && score !== null && "score" in score && typeof score.score === "number") {
+    return score.score;
+  }
+  return outcome.outcome === "pass" ? 1 : 0;
+}
+function traceFromCarrier(value) {
+  if (typeof value !== "object" || value === null || !("trace" in value)) return void 0;
+  return readTraceRef(value.trace);
+}
+function traceFromMetadata(metadata) {
+  if (metadata === void 0) return void 0;
+  return readTraceRef({
+    traceId: metadata.traceId,
+    observationId: metadata.observationId,
+    responseId: metadata.responseId
+  });
+}
+function readTraceRef(value) {
+  if (typeof value !== "object" || value === null) return void 0;
+  const traceId = value.traceId;
+  if (typeof traceId !== "string" || traceId.length === 0) return void 0;
+  const observationId = value.observationId;
+  const responseId = value.responseId;
+  const trace = { traceId };
+  if (typeof observationId === "string" && observationId.length > 0) {
+    trace.observationId = observationId;
+  }
+  if (typeof responseId === "string" && responseId.length > 0) trace.responseId = responseId;
+  return trace;
+}
+
 // src/evals/runner.ts
 async function runEvalSuite(options) {
-  const startedAt = Date.now();
-  const results = await mapWithConcurrency(
-    options.cases,
-    Math.max(1, Math.trunc(options.concurrency ?? 1)),
-    (testCase) => runEvalCase(options, testCase)
-  );
+  const startedAtMs = Date.now();
+  const run = resolveRun(options, startedAtMs);
+  const reporters = options.reporters ?? [];
+  const lifecycle = {
+    run,
+    suiteName: options.name,
+    caseCount: options.cases.length,
+    metricNames: options.metrics.map((metric) => metric.name)
+  };
+  let reporterErrors;
+  try {
+    reporterErrors = await notifyRunStart(
+      reporters,
+      lifecycle,
+      options.failOnReporterError === true
+    );
+  } catch (error) {
+    await notifyRunEnd(reporters, {
+      ...lifecycle,
+      status: "failed",
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      durationMs: Date.now() - startedAtMs,
+      error
+    });
+    throw error;
+  }
+  let results;
+  try {
+    results = await runEvalCases(options, run);
+  } catch (error) {
+    await notifyRunEnd(reporters, {
+      ...lifecycle,
+      status: "failed",
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      durationMs: Date.now() - startedAtMs,
+      error
+    });
+    throw error;
+  }
   const counts = countOutcomes(results);
-  return {
+  const completedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const result = {
     name: options.name,
+    run: { ...run, completedAt },
     results,
     ...counts,
-    durationMs: Date.now() - startedAt
+    durationMs: Date.now() - startedAtMs,
+    reporterErrors
   };
+  result.reporterErrors.push(
+    ...await notifyRunEnd(
+      reporters,
+      {
+        ...lifecycle,
+        status: "completed",
+        completedAt,
+        durationMs: result.durationMs,
+        ...counts
+      },
+      options.failOnReporterError === true
+    )
+  );
+  return result;
 }
-async function runEvalCase(options, testCase) {
+async function runEvalCases(options, run) {
+  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 1));
+  const results = new Array(options.cases.length);
+  let nextIndex = 0;
+  let failure;
+  async function worker() {
+    while (failure === void 0 && nextIndex < options.cases.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      try {
+        results[index] = await runEvalCase(
+          options,
+          options.cases[index],
+          run
+        );
+      } catch (error) {
+        failure ??= { error };
+      }
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, options.cases.length) }, () => worker())
+  );
+  if (failure !== void 0) throw failure.error;
+  return results;
+}
+async function runEvalCase(options, testCase, run) {
   let output;
   let targetError;
   try {
@@ -276,16 +1309,20 @@ async function runEvalCase(options, testCase) {
   } catch (error) {
     targetError = error;
   }
+  const traceResult = await resolveTrace(options, testCase, output, targetError);
   const metrics = [];
   for (const metric of options.metrics) {
     const outcome = targetError === void 0 ? await safeEvaluate(options.name, testCase, output, metric) : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`);
     const reporterErrors = await reportOutcome({
+      run,
       suiteName: options.name,
       testCase,
       output,
       targetError,
       metric,
       outcome,
+      trace: traceResult.trace,
+      traceError: traceResult.error,
       reporters: options.reporters ?? [],
       failOnReporterError: options.failOnReporterError === true
     });
@@ -303,6 +1340,20 @@ async function runEvalCase(options, testCase) {
   }
   return result;
 }
+async function resolveTrace(options, testCase, output, targetError) {
+  try {
+    const selector = options.trace ?? defaultEvalTraceSelector;
+    const trace = await selector({
+      suiteName: options.name,
+      case: testCase,
+      output,
+      targetError
+    });
+    return trace === void 0 ? {} : { trace };
+  } catch (error) {
+    return { error };
+  }
+}
 async function safeEvaluate(suiteName, testCase, output, metric) {
   try {
     return await metric.evaluate({ suiteName, case: testCase, output });
@@ -312,13 +1363,20 @@ async function safeEvaluate(suiteName, testCase, output, metric) {
 }
 async function reportOutcome(args) {
   const errors = [];
+  if (args.traceError !== void 0) {
+    if (args.failOnReporterError) throw args.traceError;
+    errors.push(args.traceError);
+    return errors;
+  }
   for (const reporter of args.reporters) {
     try {
       await reporter.report({
+        run: args.run,
         suiteName: args.suiteName,
         case: args.testCase,
         output: args.output,
         targetError: args.targetError,
+        trace: args.trace,
         metric: args.metric,
         outcome: args.outcome
       });
@@ -331,6 +1389,53 @@ async function reportOutcome(args) {
   }
   return errors;
 }
+function resolveRun(options, startedAtMs) {
+  const id = options.run?.id ?? globalThis.crypto.randomUUID();
+  if (id.trim().length === 0 || id.length > 128) {
+    throw new TypeError("Evaluation run id must contain 1 to 128 characters");
+  }
+  for (const [label, value] of [
+    ["dataset name", options.run?.datasetName],
+    ["dataset version", options.run?.datasetVersion]
+  ]) {
+    if (value !== void 0 && (value.trim().length === 0 || value.length > 256)) {
+      throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);
+    }
+  }
+  return {
+    id,
+    startedAt: new Date(startedAtMs).toISOString(),
+    ...options.run?.datasetName === void 0 ? {} : { datasetName: options.run.datasetName },
+    ...options.run?.datasetVersion === void 0 ? {} : { datasetVersion: options.run.datasetVersion },
+    ...options.run?.metadata === void 0 ? {} : { metadata: options.run.metadata }
+  };
+}
+async function notifyRunStart(reporters, args, failOnReporterError) {
+  const errors = [];
+  for (const reporter of reporters) {
+    if (reporter.onRunStart === void 0) continue;
+    try {
+      await reporter.onRunStart(args);
+    } catch (error) {
+      if (failOnReporterError) throw error;
+      errors.push(error);
+    }
+  }
+  return errors;
+}
+async function notifyRunEnd(reporters, args, failOnReporterError = false) {
+  const errors = [];
+  for (const reporter of reporters) {
+    if (reporter.onRunEnd === void 0) continue;
+    try {
+      await reporter.onRunEnd(args);
+    } catch (error) {
+      if (failOnReporterError) throw error;
+      errors.push(error);
+    }
+  }
+  return errors;
+}
 function countOutcomes(results) {
   let passed = 0;
   let failed = 0;
@@ -347,12 +1452,24 @@ function countOutcomes(results) {
 export {
   EvalOutcome,
   agentEvalTarget,
+  answerRelevancy,
   contains,
+  defaultEvalTraceSelector,
   defineMetric,
   exactMatch,
+  faithfulness,
+  gEval,
+  hallucination,
+  jsonCorrectness,
+  knowledgeRetention,
   llmJudge,
   llmScore,
+  projectEvalOutcome,
+  promptAlignment,
+  resolveEvalTraceRef,
   runEvalSuite,
-  semanticSimilarity
+  semanticSimilarity,
+  summarization,
+  turnRelevancy
 };
 //# sourceMappingURL=index.js.map