@anvia/core 0.23.0 → 0.25.0

package/dist/evals/index.d.ts

@@ -1,5 +1,5 @@
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
-import { l as JsonValue, C as CompletionModel, M as Message } from '../types-G12i56s2.js';
+import { U as Usage, l as JsonValue, C as CompletionModel, M as Message } from '../types-G12i56s2.js';
 import { A as Agent } from '../agent-x3YTyv5i.js';
 import { P as PromptResponse } from '../index-BxSbmOMr.js';
 import { E as EmbeddingModel } from '../types-BCTRUGex.js';
@@ -16,35 +16,54 @@ type EvalOutcome<Score = unknown> = {
     score?: Score | undefined;
     comment?: string | undefined;
     metadata?: EvalMetadata | undefined;
+    usage?: Usage | undefined;
 } | {
     outcome: "fail";
     score?: Score | undefined;
     comment?: string | undefined;
     metadata?: EvalMetadata | undefined;
+    usage?: Usage | undefined;
 } | {
     outcome: "invalid";
     reason: string;
     score?: Score | undefined;
     comment?: string | undefined;
     metadata?: EvalMetadata | undefined;
+    usage?: Usage | undefined;
 };
 declare const EvalOutcome: {
     pass<Score>(score?: Score, options?: {
         comment?: string | undefined;
         metadata?: EvalMetadata | undefined;
+        usage?: Usage | undefined;
     }): EvalOutcome<Score>;
     fail<Score>(score?: Score, options?: {
         comment?: string | undefined;
         metadata?: EvalMetadata | undefined;
+        usage?: Usage | undefined;
     }): EvalOutcome<Score>;
     invalid<Score = never>(reason: string, options?: {
         score?: Score | undefined;
         comment?: string | undefined;
         metadata?: EvalMetadata | undefined;
+        usage?: Usage | undefined;
     }): EvalOutcome<Score>;
 };
 
 type EvalMetadata = Record<string, JsonValue | undefined>;
+type EvalRunOptions = {
+    id?: string | undefined;
+    datasetName?: string | undefined;
+    datasetVersion?: string | undefined;
+    metadata?: EvalMetadata | undefined;
+};
+type EvalRunContext = {
+    id: string;
+    startedAt: string;
+    datasetName?: string | undefined;
+    datasetVersion?: string | undefined;
+    metadata?: EvalMetadata | undefined;
+};
 type EvalCase<Input, Expected = unknown> = {
     id: string;
     input: Input;
@@ -65,6 +84,36 @@ type EvalTraceRef = {
 };
 type EvalTarget<Input, Output, Expected = unknown> = (input: Input, testCase: EvalCase<Input, Expected>) => Output | Promise<Output>;
 type EvalOutcomeStatus = "pass" | "fail" | "invalid";
+type EvalScoreDirection = "higher_is_better" | "lower_is_better";
+type EvalDataType = "NUMERIC" | "CATEGORICAL" | "BOOLEAN";
+type EvalTraceCarrier = {
+    trace: EvalTraceRef;
+};
+type DefaultEvalActual<Output> = Output extends {
+    output: infer Text extends string;
+} ? Text : Output;
+type EvalCaseRequirements = {
+    expected?: unknown;
+    context?: string[];
+    retrievalContext?: string[];
+};
+type EvalTotals = {
+    total: number;
+    passed: number;
+    failed: number;
+    invalid: number;
+};
+type EvalUsageSummary = {
+    target: Usage;
+    evaluation: Usage;
+    total: Usage;
+};
+type EvalCostSummary = {
+    currency: string;
+    target: number;
+    evaluation: number;
+    total: number;
+};
 type EvalScoreProjection = {
     outcome: EvalOutcomeStatus;
     value: number | string;
@@ -78,42 +127,87 @@ type EvalMetricArgs<Input, Output, Expected = unknown> = {
     case: EvalCase<Input, Expected>;
     output: Output;
 };
-type EvalMetric<Input, Output, Score = unknown, Expected = unknown> = {
-    name: string;
-    dataType?: "NUMERIC" | "CATEGORICAL" | "BOOLEAN" | undefined;
+type EvalMetric<Input, Output, Score = unknown, Expected = unknown, Name extends string = string, Requirements extends EvalCaseRequirements = Record<never, never>> = {
+    name: Name;
+    required?: boolean | undefined;
+    direction?: EvalScoreDirection | undefined;
+    threshold?: number | undefined;
+    dataType?: EvalDataType | undefined;
+    projectScore?(score: Score): number | string | boolean;
     scoreConfigId?: string | undefined;
     configId?: string | undefined;
     metadata?: EvalMetadata | undefined;
+    readonly caseRequirements?: Requirements | undefined;
     evaluate(args: EvalMetricArgs<Input, Output, Expected>): EvalOutcome<Score> | Promise<EvalOutcome<Score>>;
 };
-type EvalMetricResult<Score = unknown> = {
-    metricName: string;
+type EvalMetricResult<Score = unknown, Name extends string = string> = {
+    metricName: Name;
+    required: boolean;
+    direction?: EvalScoreDirection | undefined;
+    threshold?: number | undefined;
     outcome: EvalOutcome<Score>;
     reporterErrors: unknown[];
 };
-type EvalCaseResult<Input, Output, Expected = unknown> = {
+type AnyEvalMetric = EvalMetric<never, never, unknown, never, string>;
+type EvalMetricScore<Metric> = Metric extends {
+    evaluate(...args: never[]): infer Result;
+} ? Awaited<Result> extends EvalOutcome<infer Score> ? Score : never : never;
+type EvalMetricResultFor<Metric> = Metric extends {
+    name: infer Name extends string;
+} ? EvalMetricResult<EvalMetricScore<Metric>, Name> : never;
+type EvalScoreMap<Metrics extends readonly AnyEvalMetric[]> = {
+    [Metric in Metrics[number] as Metric["name"]]: EvalOutcome<EvalMetricScore<Metric>>;
+};
+type EvalCaseResult<Input, Output, Expected = unknown, Metrics extends readonly AnyEvalMetric[] = readonly AnyEvalMetric[]> = {
     case: EvalCase<Input, Expected>;
+    outcome: EvalOutcomeStatus;
     output?: Output | undefined;
     targetError?: unknown;
-    metrics: EvalMetricResult[];
+    metrics: Array<EvalMetricResultFor<Metrics[number]>>;
+    scores: EvalScoreMap<Metrics>;
 };
-type EvalSuiteResult<Input, Output, Expected = unknown> = {
+type EvalSuiteResult<Input, Output, Expected = unknown, Metrics extends readonly AnyEvalMetric[] = readonly AnyEvalMetric[]> = {
     name: string;
-    results: Array<EvalCaseResult<Input, Output, Expected>>;
-    passed: number;
-    failed: number;
-    invalid: number;
+    run: EvalRunContext & {
+        completedAt: string;
+    };
+    results: Array<EvalCaseResult<Input, Output, Expected, Metrics>>;
+    metrics: EvalTotals;
+    cases: EvalTotals;
+    usage: EvalUsageSummary;
+    cost?: EvalCostSummary | undefined;
     durationMs: number;
+    reporterErrors: unknown[];
+};
+type EvalMetricDescriptor<Score = unknown, Name extends string = string> = Omit<EvalMetric<never, never, Score, never, Name>, "evaluate"> & {
+    evaluate?: EvalMetric<never, never, Score, never, Name>["evaluate"];
 };
 type EvalReportArgs<Input, Output, Score = unknown, Expected = unknown> = {
+    run?: EvalRunContext | undefined;
     suiteName: string;
     case: EvalCase<Input, Expected>;
     output?: Output | undefined;
     targetError?: unknown;
     trace?: EvalTraceRef | undefined;
-    metric: EvalMetric<Input, Output, Score, Expected>;
+    metric: EvalMetricDescriptor<Score>;
     outcome: EvalOutcome<Score>;
 };
+type EvalRunStartArgs = {
+    run: EvalRunContext;
+    suiteName: string;
+    caseCount: number;
+    metricNames: string[];
+};
+type EvalRunEndArgs = EvalRunStartArgs & {
+    status: "completed" | "failed";
+    completedAt: string;
+    durationMs: number;
+    metrics?: EvalTotals | undefined;
+    cases?: EvalTotals | undefined;
+    usage?: EvalUsageSummary | undefined;
+    cost?: EvalCostSummary | undefined;
+    error?: unknown;
+};
 type EvalTraceSelectorArgs<Input, Output, Expected = unknown> = {
     suiteName: string;
     case: EvalCase<Input, Expected>;
@@ -121,24 +215,49 @@ type EvalTraceSelectorArgs<Input, Output, Expected = unknown> = {
     targetError?: unknown;
 };
 type EvalTraceSelector<Input, Output, Expected = unknown> = (args: EvalTraceSelectorArgs<Input, Output, Expected>) => EvalTraceRef | undefined | Promise<EvalTraceRef | undefined>;
-type EvalReporter<Input = unknown, Output = unknown, Expected = unknown> = {
+type EvalReporter<in Input = unknown, in Output = unknown, in Expected = unknown> = {
+    onRunStart?(args: EvalRunStartArgs): void | Promise<void>;
     report(args: EvalReportArgs<Input, Output, unknown, Expected>): void | Promise<void>;
+    onRunEnd?(args: EvalRunEndArgs): void | Promise<void>;
+};
+type EvalTargetUsageSelector<Input, Output, Expected = unknown> = (args: EvalMetricArgs<Input, Output, Expected>) => Usage | undefined | Promise<Usage | undefined>;
+type EvalCostCalculatorArgs<Input, Output, Expected = unknown> = {
+    kind: "target";
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    output: Output;
+    usage: Usage;
+} | {
+    kind: "evaluation";
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    output: Output;
+    metric: EvalMetric<Input, Output, unknown, Expected>;
+    usage: Usage;
+};
+type EvalCostOptions<Input, Output, Expected = unknown> = {
+    currency: string;
+    calculate(args: EvalCostCalculatorArgs<Input, Output, Expected>): number | Promise<number>;
 };
-type RunEvalSuiteOptions<Input, Output, Expected = unknown> = {
+type RunEvalSuiteOptions<Input, Output, Expected = unknown, Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]> = {
     name: string;
-    cases: Array<EvalCase<Input, Expected>>;
+    run?: EvalRunOptions | undefined;
+    cases: readonly EvalCase<Input, Expected>[];
     target: EvalTarget<Input, Output, Expected>;
-    metrics: Array<EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>>>;
+    metrics: Metrics;
     concurrency?: number | undefined;
     trace?: EvalTraceSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
-    reporters?: Array<EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>> | undefined;
+    reporters?: readonly EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>[] | undefined;
     failOnReporterError?: boolean | undefined;
+    targetUsage?: EvalTargetUsageSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
+    cost?: EvalCostOptions<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
 };
 type ValueSelector<Input, Output, Expected, Value> = (args: EvalMetricArgs<Input, Output, Expected>) => Value | Promise<Value>;
 type SelectorOrValue<Input, Output, Expected, Value> = Value | ValueSelector<Input, Output, Expected, Value>;
 
 type LlmEvalOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     model: CompletionModel;
     threshold?: number | undefined;
     strictMode?: boolean | undefined;
@@ -148,13 +267,18 @@ type LlmEvalOptions<Input, Output, Expected = unknown> = {
     actual?: ValueSelector<Input, Output, Expected, string> | undefined;
 };
 type AnswerRelevancyOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected>;
-declare function answerRelevancy<Input, Output, Expected = unknown>(options: AnswerRelevancyOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function answerRelevancy<Input, Output, Expected = unknown, const Name extends string = string>(options: AnswerRelevancyOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type PromptAlignmentOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
     promptInstructions: string[];
 };
-declare function promptAlignment<Input, Output, Expected = unknown>(options: PromptAlignmentOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function promptAlignment<Input, Output, Expected = unknown, const Name extends string = string>(options: PromptAlignmentOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     schema: ZodSchema<SchemaOutput>;
     model?: CompletionModel | undefined;
     threshold?: number | undefined;
@@ -163,23 +287,59 @@ type JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected = unknown> = {
     retries?: number | undefined;
     actual?: ValueSelector<Input, Output, Expected, string> | undefined;
 };
-declare function jsonCorrectness<Input, Output, SchemaOutput, Expected = unknown>(options: JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function jsonCorrectness<Input, Output, SchemaOutput, Expected = unknown, const Name extends string = string>(options: JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type HallucinationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
     context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
 };
-declare function hallucination<Input, Output, Expected = unknown>(options: HallucinationOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function hallucination<Input, Output, Expected = unknown, const Name extends string = string>(options: HallucinationOptions<Input, Output, Expected> & {
+    context: Exclude<HallucinationOptions<Input, Output, Expected>["context"], undefined>;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
+declare function hallucination<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<HallucinationOptions<Input, Output, Expected>, "context"> & {
+    context?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name, {
+    context: string[];
+}>;
 type FaithfulnessOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
     retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
     truthsExtractionLimit?: number | undefined;
     penalizeAmbiguousClaims?: boolean | undefined;
 };
-declare function faithfulness<Input, Output, Expected = unknown>(options: FaithfulnessOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function faithfulness<Input, Output, Expected = unknown, const Name extends string = string>(options: FaithfulnessOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+    retrievalContext: Exclude<FaithfulnessOptions<Input, Output, Expected>["retrievalContext"], undefined>;
+}): EvalMetric<Input, Output, number, Expected, Name>;
+declare function faithfulness<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<FaithfulnessOptions<Input, Output, Expected>, "retrievalContext"> & {
+    name?: Name | undefined;
+    retrievalContext?: undefined;
+}): EvalMetric<Input, Output, number, Expected, Name, {
+    retrievalContext: string[];
+}>;
+type AbstentionCategory = "correct_abstention" | "unnecessary_abstention" | "unsupported_confident_answer" | "correct_grounded_answer";
+type AbstentionOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    required?: boolean | undefined;
+    model: CompletionModel;
+    shouldAbstain: SelectorOrValue<Input, Output, Expected, boolean>;
+    context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+    includeReason?: boolean | undefined;
+    retries?: number | undefined;
+};
+declare function abstention<Input, Output, Expected = unknown, const Name extends string = string>(options: AbstentionOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, AbstentionCategory, Expected, Name>;
 type SummarizationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<Input, Output, Expected> & {
     assessmentQuestions?: string[] | undefined;
     questionCount?: number | undefined;
     truthsExtractionLimit?: number | undefined;
 };
-declare function summarization<Input, Output, Expected = unknown>(options: SummarizationOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function summarization<Input, Output, Expected = unknown, const Name extends string = string>(options: SummarizationOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type GEvalParameter = "input" | "actualOutput" | "expectedOutput" | "context" | "retrievalContext" | "metadata";
 type GEvalRubric = {
     scoreRange: readonly [number, number];
@@ -195,9 +355,12 @@ type GEvalOptions<Input, Output, Expected = unknown> = Omit<LlmEvalOptions<Input
     context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
     retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
 };
-declare function gEval<Input, Output, Expected = unknown>(options: GEvalOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function gEval<Input, Output, Expected = unknown, const Name extends string = string>(options: GEvalOptions<Input, Output, Expected> & {
+    name: Name;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type ConversationEvalOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     model: CompletionModel;
     threshold?: number | undefined;
     strictMode?: boolean | undefined;
@@ -209,42 +372,160 @@ type ConversationEvalOptions<Input, Output, Expected = unknown> = {
 type TurnRelevancyOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<Input, Output, Expected> & {
     windowSize?: number | undefined;
 };
-declare function turnRelevancy<Input, Output, Expected = unknown>(options: TurnRelevancyOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function turnRelevancy<Input, Output, Expected = unknown, const Name extends string = string>(options: TurnRelevancyOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type KnowledgeRetentionOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<Input, Output, Expected>;
-declare function knowledgeRetention<Input, Output, Expected = unknown>(options: KnowledgeRetentionOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function knowledgeRetention<Input, Output, Expected = unknown, const Name extends string = string>(options: KnowledgeRetentionOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
 type ConversationSource = EvalTurn[] | Message[];
 
-type AgentEvalTargetOptions<Input, Output = PromptResponse> = {
-    prompt?: ((input: Input, testCase: EvalCase<Input>) => string | Message) | undefined;
-    output?: ((response: PromptResponse, testCase: EvalCase<Input>) => Output) | undefined;
+type AgentEvalTargetOptions<Input, Output = PromptResponse, Expected = unknown> = {
+    prompt?: ((input: Input, testCase: EvalCase<Input, Expected>) => string | Message) | undefined;
+    output?: ((response: PromptResponse, testCase: EvalCase<Input, Expected>) => Output) | undefined;
 };
-declare function agentEvalTarget<Input>(agent: Agent, options?: AgentEvalTargetOptions<Input, PromptResponse>): EvalTarget<Input, PromptResponse>;
-declare function agentEvalTarget<Input, Output>(agent: Agent, options: AgentEvalTargetOptions<Input, Output>): EvalTarget<Input, Output>;
+declare function agentEvalTarget<Input, Expected = unknown>(agent: Agent, options?: AgentEvalTargetOptions<Input, PromptResponse, Expected>): EvalTarget<Input, PromptResponse, Expected>;
+declare function agentEvalTarget<Input, Output, Expected = unknown>(agent: Agent, options: AgentEvalTargetOptions<Input, Output, Expected>): EvalTarget<Input, Output, Expected>;
+
+type EvalOutputFormat = "pretty" | "json" | "quiet";
+type EvalExpectedTotals = Partial<EvalTotals> & {
+    metrics?: Partial<EvalTotals> | undefined;
+    cases?: Partial<EvalTotals> | undefined;
+};
+type EvalExpectedOutcomes = Record<string, Record<string, EvalOutcomeStatus>>;
+type EvalExpectations = {
+    totals?: EvalExpectedTotals | undefined;
+    outcomes?: EvalExpectedOutcomes | undefined;
+};
+type EvalOutputWriters = {
+    stdout?(text: string): void;
+    stderr?(text: string): void;
+};
+type PrintEvalResultOptions = {
+    format?: EvalOutputFormat | undefined;
+    output?: EvalOutputWriters | undefined;
+};
+type RunEvalCliOptions<Input, Output, Expected = unknown, Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]> = RunEvalSuiteOptions<Input, Output, Expected, Metrics> & {
+    format?: EvalOutputFormat | undefined;
+    exitCode?: boolean | undefined;
+    expectations?: EvalExpectations | undefined;
+    output?: EvalOutputWriters | undefined;
+};
+declare class EvalAssertionError extends Error {
+    readonly mismatches: string[];
+    constructor(message: string, mismatches: string[]);
+}
+declare function printEvalResult(result: EvalSuiteResult<unknown, unknown, unknown>, options?: PrintEvalResultOptions): void;
+declare function evalExitCode(result: EvalSuiteResult<unknown, unknown, unknown>, expectations?: EvalExpectations): 0 | 1 | 2;
+declare function assertEvalTotals(result: EvalSuiteResult<unknown, unknown, unknown>, expected: EvalExpectedTotals): void;
+declare function assertEvalOutcomes(result: EvalSuiteResult<unknown, unknown, unknown>, expected: EvalExpectedOutcomes): void;
+declare function runEvalCli<Input, Output, Expected = unknown, const Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]>(options: RunEvalCliOptions<Input, Output, Expected, Metrics>): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;
 
-declare function defineMetric<Input, Output, Score, Expected>(metric: EvalMetric<Input, Output, Score, Expected>): EvalMetric<Input, Output, Score, Expected>;
+declare function defineMetric<Input, Output, Score, Expected, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name>): EvalMetric<Input, Output, Score, Expected, Name>;
 
 type ExactMatchOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;
     expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;
 };
-declare function exactMatch<Input, Output, Expected = unknown>(options?: ExactMatchOptions<Input, Output, Expected>): EvalMetric<Input, Output, boolean, Expected>;
+declare function exactMatch<Input, Output, Expected = unknown, const Name extends string = string>(options: ExactMatchOptions<Input, Output, Expected> & {
+    expected: Exclude<ExactMatchOptions<Input, Output, Expected>["expected"], undefined>;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function exactMatch<Input, Output, Expected = unknown, const Name extends string = string>(options?: Omit<ExactMatchOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: unknown;
+}>;
 type ContainsOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     actual?: ValueSelector<Input, Output, Expected, string> | undefined;
     expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;
 };
-declare function contains<Input, Output, Expected = unknown>(options?: ContainsOptions<Input, Output, Expected>): EvalMetric<Input, Output, boolean, Expected>;
+declare function contains<Input, Output, Expected = unknown, const Name extends string = string>(options: ContainsOptions<Input, Output, Expected> & {
+    expected: Exclude<ContainsOptions<Input, Output, Expected>["expected"], undefined>;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function contains<Input, Output, Expected = unknown, const Name extends string = string>(options?: Omit<ContainsOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: string | RegExp;
+}>;
+type NotContainsOptions<Input, Output, Expected = unknown> = ContainsOptions<Input, Output, Expected>;
+declare function notContains<Input, Output, Expected = unknown, const Name extends string = string>(options?: NotContainsOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type ContainsListOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    required?: boolean | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+    expected?: SelectorOrValue<Input, Output, Expected, ReadonlyArray<string | RegExp>> | undefined;
+};
+type ContainsAllOptions<Input, Output, Expected = unknown> = ContainsListOptions<Input, Output, Expected>;
+declare function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(options: ContainsAllOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type ContainsAnyOptions<Input, Output, Expected = unknown> = ContainsListOptions<Input, Output, Expected>;
+declare function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(options: ContainsAnyOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type MatchesOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    required?: boolean | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+    expected?: SelectorOrValue<Input, Output, Expected, RegExp> | undefined;
+};
+declare function matches<Input, Output, Expected = unknown, const Name extends string = string>(options: MatchesOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type DoesNotMatchOptions<Input, Output, Expected = unknown> = MatchesOptions<Input, Output, Expected>;
+declare function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(options: DoesNotMatchOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type MaxLengthOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    required?: boolean | undefined;
+    actual?: ValueSelector<Input, Output, Expected, string> | undefined;
+    max: SelectorOrValue<Input, Output, Expected, number>;
+};
+declare function maxLength<Input, Output, Expected = unknown, const Name extends string = string>(options: MaxLengthOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
+type RequiredFieldsOptions<Input, Output, Expected = unknown> = {
+    name?: string | undefined;
+    required?: boolean | undefined;
+    actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;
+    expected: SelectorOrValue<Input, Output, Expected, readonly string[]>;
+};
+declare function requiredFields<Input, Output, Expected = unknown, const Name extends string = string>(options: RequiredFieldsOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name>;
 type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     model: EmbeddingModel;
     threshold: number;
     actual?: ValueSelector<Input, Output, Expected, string> | undefined;
     expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;
 };
-declare function semanticSimilarity<Input, Output, Expected = unknown>(options: SemanticSimilarityOptions<Input, Output, Expected>): EvalMetric<Input, Output, number, Expected>;
+declare function semanticSimilarity<Input, Output, Expected = unknown, const Name extends string = string>(options: SemanticSimilarityOptions<Input, Output, Expected> & {
+    expected: Exclude<SemanticSimilarityOptions<Input, Output, Expected>["expected"], undefined>;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name>;
+declare function semanticSimilarity<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<SemanticSimilarityOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, number, Expected, Name, {
+    expected: string;
+}>;
 type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     model: CompletionModel;
     schema: ZodSchema<SchemaOutput>;
     passes(value: SchemaOutput): boolean;
@@ -252,13 +533,16 @@ type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {
     retries?: number | undefined;
     prompt?: ValueSelector<Input, Output, Expected, string> | undefined;
 };
-declare function llmJudge<Input, Output, SchemaOutput, Expected = unknown>(options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected>): EvalMetric<Input, Output, SchemaOutput, Expected>;
+declare function llmJudge<Input, Output, SchemaOutput, Expected = unknown, const Name extends string = string>(options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, SchemaOutput, Expected, Name>;
 type LlmScoreMetricScore = {
     score: number;
     feedback: string;
 };
 type LlmScoreOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
+    required?: boolean | undefined;
     model: CompletionModel;
     threshold: number;
     criteria: string | string[];
@@ -266,9 +550,11 @@ type LlmScoreOptions<Input, Output, Expected = unknown> = {
     retries?: number | undefined;
     prompt?: ValueSelector<Input, Output, Expected, string> | undefined;
 };
-declare function llmScore<Input, Output, Expected = unknown>(options: LlmScoreOptions<Input, Output, Expected>): EvalMetric<Input, Output, LlmScoreMetricScore, Expected>;
+declare function llmScore<Input, Output, Expected = unknown, const Name extends string = string>(options: LlmScoreOptions<Input, Output, Expected> & {
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, LlmScoreMetricScore, Expected, Name>;
 
-declare function projectEvalOutcome(outcome: EvalOutcome, dataType: EvalMetric<unknown, unknown>["dataType"]): EvalScoreProjection;
+declare function projectEvalOutcome<Score>(outcome: EvalOutcome<Score>, dataType: EvalMetric<unknown, unknown>["dataType"], projectScore?: ((score: Score) => number | string | boolean) | undefined): EvalScoreProjection;
 declare function resolveEvalTraceRef(args: {
     output?: unknown;
     input?: unknown;
@@ -276,6 +562,75 @@ declare function resolveEvalTraceRef(args: {
 }): EvalTraceRef | undefined;
 declare function defaultEvalTraceSelector<Input, Output, Expected>(args: EvalTraceSelectorArgs<Input, Output, Expected>): EvalTraceRef | undefined;
 
-declare function runEvalSuite<Input, Output, Expected = unknown>(options: RunEvalSuiteOptions<Input, Output, Expected>): Promise<EvalSuiteResult<Input, Output, Expected>>;
+declare function runEvalSuite<Input, Output, Expected = unknown, const Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]>(options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;
+
+declare function selectPromptOutput(args: EvalMetricArgs<unknown, unknown, unknown>): string;
+
+type EvalCaseLike = EvalCase<unknown, unknown>;
+type EvalCasesInput<Cases extends readonly EvalCaseLike[]> = Cases[number]["input"];
+type EvalCaseExpected<Case> = Case extends {
+    expected: infer Expected;
+} ? Expected : unknown;
+type EvalCasesExpected<Cases extends readonly EvalCaseLike[]> = EvalCaseExpected<Cases[number]>;
+type EvalMetricRequirements<Metric> = Metric extends EvalMetric<infer _Input, infer _Output, infer _Score, infer _Expected, infer _Name, infer Requirements> ? Requirements : Record<never, never>;
+type RequiredExpected<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    expected: unknown;
+}>;
+type RequiredContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    context: string[];
+}>;
+type RequiredRetrievalContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    retrievalContext: string[];
+}>;
+type EvalCaseFieldsForMetrics<Metrics extends readonly EvalMetric<never, never>[]> = ([
+    RequiredExpected<Metrics>
+] extends [never] ? Record<never, never> : {
+    expected: RequiredExpected<Metrics>["expected"];
+}) & ([RequiredContext<Metrics>] extends [never] ? Record<never, never> : {
+    context: string[];
+}) & ([RequiredRetrievalContext<Metrics>] extends [never] ? Record<never, never> : {
+    retrievalContext: string[];
+});
+type EvalCasesForMetrics<Cases extends readonly EvalCaseLike[], Metrics extends readonly EvalMetric<never, never>[]> = {
+    readonly [Index in keyof Cases]: Cases[Index] & EvalCaseFieldsForMetrics<Metrics>;
+};
+type DefinedEvalSuite<Cases extends readonly EvalCaseLike[], Output, Metrics extends readonly EvalMetric<EvalCasesInput<Cases>, Output, unknown, EvalCasesExpected<Cases>, string>[]> = Omit<RunEvalSuiteOptions<EvalCasesInput<Cases>, Output, EvalCasesExpected<Cases>, Metrics>, "cases" | "target" | "metrics"> & {
+    cases: Cases & EvalCasesForMetrics<NoInfer<Cases>, NoInfer<Metrics>>;
+    target: EvalTarget<EvalCasesInput<Cases>, Output, EvalCasesExpected<Cases>>;
+    metrics: Metrics;
+};
+type EvalSuiteTypeBuilder<Input, Output, Expected = unknown> = {
+    defineMetric<const Name extends string = string>(metric: EvalMetric<Input, Output, boolean, Expected, Name> & {
+        dataType: "BOOLEAN";
+    }): EvalMetric<Input, Output, boolean, Expected, Name>;
+    defineMetric<const Name extends string = string>(metric: EvalMetric<Input, Output, number, Expected, Name> & {
+        dataType: "NUMERIC";
+    }): EvalMetric<Input, Output, number, Expected, Name>;
+    defineMetric<const Name extends string = string>(metric: EvalMetric<Input, Output, string, Expected, Name> & {
+        dataType: "CATEGORICAL";
+    }): EvalMetric<Input, Output, string, Expected, Name>;
+    defineMetric<Score, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name> & {
+        dataType: "BOOLEAN";
+        projectScore(score: Score): boolean;
+    }): EvalMetric<Input, Output, Score, Expected, Name>;
+    defineMetric<Score, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name> & {
+        dataType: "NUMERIC";
+        projectScore(score: Score): number;
+    }): EvalMetric<Input, Output, Score, Expected, Name>;
+    defineMetric<Score, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name> & {
+        dataType: "CATEGORICAL";
+        projectScore(score: Score): string;
+    }): EvalMetric<Input, Output, Score, Expected, Name>;
+    defineMetric<Score, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name> & {
+        dataType?: undefined;
+    }): EvalMetric<Input, Output, Score, Expected, Name>;
+};
+declare function defineEvalCases<const Cases extends readonly EvalCaseLike[]>(cases: Cases): Cases;
+declare function defineEvalSuite<Input, Output, Expected = unknown>(): EvalSuiteTypeBuilder<Input, Output, Expected>;
+declare function defineEvalSuite<const Cases extends readonly EvalCaseLike[], const Target extends (input: EvalCasesInput<Cases>, testCase: EvalCase<EvalCasesInput<Cases>, EvalCasesExpected<Cases>>) => unknown, const Metrics extends readonly EvalMetric<EvalCasesInput<Cases>, Awaited<ReturnType<Target>>, unknown, EvalCasesExpected<Cases>, string>[]>(options: DefinedEvalSuite<Cases, Awaited<ReturnType<Target>>, Metrics> & {
+    target: Target;
+}): DefinedEvalSuite<Cases, Awaited<ReturnType<Target>>, Metrics> & {
+    target: Target;
+};
 
-export { type AgentEvalTargetOptions, type AnswerRelevancyOptions, type ContainsOptions, type EvalCase, type EvalCaseResult, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricResult, EvalOutcome, type EvalOutcomeStatus, type EvalReportArgs, type EvalReporter, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type PromptAlignmentOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, agentEvalTarget, answerRelevancy, contains, defaultEvalTraceSelector, defineMetric, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, projectEvalOutcome, promptAlignment, resolveEvalTraceRef, runEvalSuite, semanticSimilarity, summarization, turnRelevancy };
+export { type AbstentionCategory, type AbstentionOptions, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalSuiteTypeBuilder, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };