@anvia/core 1.0.4 → 1.0.8

package/dist/evals/index.d.ts

@@ -1,17 +1,22 @@
 import { U as Usage, J as JsonObject, c as CompletionModel, M as Message } from '../types-DgvozfPc.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
 import { AgentInteractionRequest, AgentInteractionResponse } from '../agent/interactions/index.js';
-import { g as AgentInteractionOutcome, m as AgentResponse, n as AgentRunOptions, k as AgentOutcome } from '../types-ChH_bdtR.js';
+import { g as AgentInteractionOutcome, m as AgentResponse, n as AgentRunOptions, k as AgentOutcome } from '../types-DL7wX6hJ.js';
 import { E as EmbeddingModel } from '../types-Cr4uiYo5.js';
 import '../model-call-options-CZkSw_xN.js';
 import 'zod';
 import '../retry-CjvSlKGW.js';
 import '../types-DhkodEft.js';
 import '../type-utils-CtHVDRn_.js';
-import '../types-CSYt7-It.js';
+import '../types-DmzRCbU5.js';
 import '../tool-BpqpoRSE.js';
 import '../middleware-kcF8AusP.js';
 
+type EvalOutcomeOptions = {
+    comment?: string | undefined;
+    metadata?: EvalMetadata | undefined;
+    usage?: Usage | undefined;
+};
 type EvalOutcome<Score = unknown> = {
     outcome: "pass";
     score?: Score | undefined;
@@ -27,32 +32,27 @@ type EvalOutcome<Score = unknown> = {
 } | {
     outcome: "invalid";
     reason: string;
+    kind?: EvalInvalidKind | undefined;
+    error?: unknown;
     score?: Score | undefined;
     comment?: string | undefined;
     metadata?: EvalMetadata | undefined;
     usage?: Usage | undefined;
 };
 declare const EvalOutcome: {
-    pass<Score>(score?: Score, options?: {
-        comment?: string | undefined;
-        metadata?: EvalMetadata | undefined;
-        usage?: Usage | undefined;
-    }): EvalOutcome<Score>;
-    fail<Score>(score?: Score, options?: {
-        comment?: string | undefined;
-        metadata?: EvalMetadata | undefined;
-        usage?: Usage | undefined;
-    }): EvalOutcome<Score>;
-    invalid<Score = never>(reason: string, options?: {
+    pass<Score>(score?: Score, options?: EvalOutcomeOptions): EvalOutcome<Score>;
+    fail<Score>(score?: Score, options?: EvalOutcomeOptions): EvalOutcome<Score>;
+    invalid<Score = never>(reason: string, options?: EvalOutcomeOptions & {
         score?: Score | undefined;
-        comment?: string | undefined;
-        metadata?: EvalMetadata | undefined;
-        usage?: Usage | undefined;
+        kind?: EvalInvalidKind | undefined;
+        error?: unknown;
     }): EvalOutcome<Score>;
+    fromError(error: unknown, kind?: EvalInvalidKind): EvalOutcome<never>;
 };
 
 type EvalMetadata = JsonObject;
 type EvalReporterErrorPolicy = "collect" | "throw";
+type EvalInvalidKind = "target" | "metric" | "configuration" | "provider" | "timeout";
 type EvalRunOptions = {
     id?: string | undefined;
     datasetName?: string | undefined;
@@ -85,7 +85,10 @@ type EvalTraceRef = {
     observationId?: string | undefined;
     responseId?: string | undefined;
 };
-type EvalTarget<Input, Output, Expected = unknown> = (input: Input, testCase: EvalCase<Input, Expected>) => Output | Promise<Output>;
+type EvalTarget<Input, Output, Expected = unknown> = (input: Input, testCase: EvalCase<Input, Expected>, context?: EvalTargetContext) => Output | Promise<Output>;
+type EvalTargetContext = {
+    signal: AbortSignal;
+};
 type EvalOutcomeStatus = "pass" | "fail" | "invalid";
 type EvalScoreDirection = "higher_is_better" | "lower_is_better";
 type EvalDataType = "NUMERIC" | "CATEGORICAL" | "BOOLEAN";
@@ -129,6 +132,7 @@ type EvalMetricArgs<Input, Output, Expected = unknown> = {
     suiteName: string;
     case: EvalCase<Input, Expected>;
     output: Output;
+    signal: AbortSignal;
 };
 type EvalMetric<Input, Output, Score = unknown, Expected = unknown, Name extends string = string, Requirements extends EvalCaseRequirements = Record<never, never>> = {
     name: Name;
@@ -149,6 +153,8 @@ type EvalMetricResult<Score = unknown, Name extends string = string> = {
     direction?: EvalScoreDirection | undefined;
     threshold?: number | undefined;
     outcome: EvalOutcome<Score>;
+    durationMs: number;
+    cost?: number | undefined;
     reporterErrors: unknown[];
 };
 type AnyEvalMetric = EvalMetric<never, never, unknown, never, string>;
@@ -164,10 +170,15 @@ type EvalScoreMap<Metrics extends readonly AnyEvalMetric[]> = {
 type EvalCaseResult<Input, Output, Expected = unknown, Metrics extends readonly AnyEvalMetric[] = readonly AnyEvalMetric[]> = {
     case: EvalCase<Input, Expected>;
     outcome: EvalOutcomeStatus;
+    targetStatus: "succeeded" | "failed";
     output?: Output | undefined;
     targetError?: unknown;
+    targetDurationMs: number;
+    durationMs: number;
     metrics: Array<EvalMetricResultFor<Metrics[number]>>;
     scores: EvalScoreMap<Metrics>;
+    usage: EvalUsageSummary;
+    cost?: EvalCostSummary | undefined;
 };
 type EvalSuiteResult<Input, Output, Expected = unknown, Metrics extends readonly AnyEvalMetric[] = readonly AnyEvalMetric[]> = {
     name: string;
@@ -191,6 +202,7 @@ type EvalReportArgs<Input, Output, Score = unknown, Expected = unknown> = {
     case: EvalCase<Input, Expected>;
     output?: Output | undefined;
     targetError?: unknown;
+    targetStatus?: "succeeded" | "failed" | undefined;
     trace?: EvalTraceRef | undefined;
     metric: EvalMetricDescriptor<Score>;
     outcome: EvalOutcome<Score>;
@@ -216,6 +228,7 @@ type EvalTraceSelectorArgs<Input, Output, Expected = unknown> = {
     case: EvalCase<Input, Expected>;
     output?: Output | undefined;
     targetError?: unknown;
+    targetStatus?: "succeeded" | "failed" | undefined;
 };
 type EvalTraceSelector<Input, Output, Expected = unknown> = (args: EvalTraceSelectorArgs<Input, Output, Expected>) => EvalTraceRef | undefined | Promise<EvalTraceRef | undefined>;
 type EvalReporter<in Input = unknown, in Output = unknown, in Expected = unknown> = {
@@ -242,13 +255,81 @@ type EvalCostOptions<Input, Output, Expected = unknown> = {
     currency: string;
     calculate(args: EvalCostCalculatorArgs<Input, Output, Expected>): number | Promise<number>;
 };
+type EvalCaseLike$1 = EvalCase<unknown, unknown>;
+type EvalMetricRequirements<Metric> = Metric extends EvalMetric<infer _Input, infer _Output, infer _Score, infer _Expected, infer _Name, infer Requirements> ? Requirements : Record<never, never>;
+type RequiredExpected<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    expected: unknown;
+}>;
+type RequiredContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    context: string[];
+}>;
+type RequiredRetrievalContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
+    retrievalContext: string[];
+}>;
+type EvalCaseFieldsForMetrics<Metrics extends readonly EvalMetric<never, never>[]> = ([
+    RequiredExpected<Metrics>
+] extends [never] ? Record<never, never> : {
+    expected: RequiredExpected<Metrics>["expected"];
+}) & ([RequiredContext<Metrics>] extends [never] ? Record<never, never> : {
+    context: string[];
+}) & ([RequiredRetrievalContext<Metrics>] extends [never] ? Record<never, never> : {
+    retrievalContext: string[];
+});
+type EvalCasesForMetrics<Cases extends readonly EvalCaseLike$1[], Metrics extends readonly EvalMetric<never, never>[]> = {
+    readonly [Index in keyof Cases]: Cases[Index] extends EvalCaseLike$1 ? Cases[Index] & EvalCaseFieldsForMetrics<Metrics> : Cases[Index];
+};
+type EvalShard = {
+    index: number;
+    count: number;
+};
+type EvalProgressEvent<Input, Output, Expected = unknown> = {
+    type: "case-start";
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    completedCases: number;
+    totalCases: number;
+} | {
+    type: "target-complete";
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    targetStatus: "succeeded" | "failed";
+    output?: Output | undefined;
+    error?: unknown;
+    durationMs: number;
+    completedCases: number;
+    totalCases: number;
+} | {
+    type: "metric-complete";
+    suiteName: string;
+    case: EvalCase<Input, Expected>;
+    metricName: string;
+    outcome: EvalOutcome<unknown>;
+    durationMs: number;
+    completedCases: number;
+    totalCases: number;
+} | {
+    type: "case-complete";
+    suiteName: string;
+    result: EvalCaseResult<Input, Output, Expected>;
+    completedCases: number;
+    totalCases: number;
+};
 type RunEvalSuiteOptions<Input, Output, Expected = unknown, Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]> = {
     name: string;
     run?: EvalRunOptions | undefined;
-    cases: readonly EvalCase<Input, Expected>[];
+    cases: readonly EvalCase<Input, Expected>[] & EvalCasesForMetrics<readonly EvalCase<Input, Expected>[], Metrics>;
     target: EvalTarget<Input, Output, Expected>;
     metrics: Metrics;
     concurrency?: number | undefined;
+    targetConcurrency?: number | undefined;
+    metricConcurrency?: number | undefined;
+    caseTimeoutMs?: number | undefined;
+    signal?: AbortSignal | undefined;
+    failFast?: boolean | undefined;
+    caseIds?: readonly string[] | undefined;
+    caseFilter?(testCase: EvalCase<Input, Expected>, index: number): boolean;
+    shard?: EvalShard | undefined;
+    onProgress?(event: EvalProgressEvent<Input, Output, Expected>): void | Promise<void>;
     trace?: EvalTraceSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
     reporters?: readonly EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>[] | undefined;
     reporterErrorPolicy?: EvalReporterErrorPolicy | undefined;
@@ -358,9 +439,20 @@ type GEvalOptions<Input, Output, Expected = unknown> = Omit<LlmEvalOptions<Input
     context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
     retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;
 };
-declare function gEval<Input, Output, Expected = unknown, const Name extends string = string>(options: GEvalOptions<Input, Output, Expected> & {
+type GEvalCaseRequirements<Params extends readonly GEvalParameter[], ExpectedSelector, ContextSelector, RetrievalContextSelector> = ("expectedOutput" extends Params[number] ? [ExpectedSelector] extends [undefined] ? {
+    expected: unknown;
+} : Record<never, never> : Record<never, never>) & ("context" extends Params[number] ? [ContextSelector] extends [undefined] ? {
+    context: string[];
+} : Record<never, never> : Record<never, never>) & ("retrievalContext" extends Params[number] ? [RetrievalContextSelector] extends [undefined] ? {
+    retrievalContext: string[];
+} : Record<never, never> : Record<never, never>);
+declare function gEval<Input, Output, Expected = unknown, const Name extends string = string, const Params extends readonly GEvalParameter[] = readonly GEvalParameter[], ExpectedSelector extends ValueSelector<Input, Output, Expected, unknown> | undefined = undefined, ContextSelector extends SelectorOrValue<Input, Output, Expected, string[]> | undefined = undefined, RetrievalContextSelector extends SelectorOrValue<Input, Output, Expected, string[]> | undefined = undefined>(options: Omit<GEvalOptions<Input, Output, Expected>, "evaluationParams" | "expected" | "context" | "retrievalContext"> & {
     name: Name;
-}): EvalMetric<Input, Output, number, Expected, Name>;
+    evaluationParams: Params;
+    expected?: ExpectedSelector;
+    context?: ContextSelector;
+    retrievalContext?: RetrievalContextSelector;
+}): EvalMetric<Input, Output, number, Expected, Name, GEvalCaseRequirements<Params, ExpectedSelector, ContextSelector, RetrievalContextSelector>>;
 type ConversationEvalOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
     required?: boolean | undefined;
@@ -434,26 +526,58 @@ type EvalOutputWriters = {
     stdout?(text: string): void;
     stderr?(text: string): void;
 };
+type EvalRedactionContext = {
+    kind: "input" | "expected" | "context" | "retrievalContext" | "output" | "score" | "comment" | "metadata" | "error";
+    caseId?: string | undefined;
+    metricName?: string | undefined;
+};
+type EvalRedactor = (value: unknown, context: EvalRedactionContext) => unknown;
 type PrintEvalResultOptions = {
     format?: EvalOutputFormat | undefined;
     output?: EvalOutputWriters | undefined;
+    maxValueLength?: number | undefined;
+    redact?: EvalRedactor | undefined;
+};
+type EvalSuiteShape = {
+    cases: readonly {
+        id: string;
+    }[];
+    metrics: readonly {
+        name: string;
+    }[];
+};
+type EvalExpectedOutcomesFor<Suite extends EvalSuiteShape> = Partial<Record<Suite["cases"][number]["id"], Partial<Record<Suite["metrics"][number]["name"], EvalOutcomeStatus>>>>;
+type EvalExpectationsFor<Suite extends EvalSuiteShape> = Omit<EvalExpectations, "outcomes"> & {
+    outcomes?: EvalExpectedOutcomesFor<Suite> | undefined;
 };
 type RunEvalCliOptions<Input, Output, Expected = unknown, Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]> = RunEvalSuiteOptions<Input, Output, Expected, Metrics> & {
     format?: EvalOutputFormat | undefined;
     exitCode?: boolean | undefined;
     expectations?: EvalExpectations | undefined;
     output?: EvalOutputWriters | undefined;
+    maxValueLength?: number | undefined;
+    redact?: EvalRedactor | undefined;
 };
 declare class EvalAssertionError extends Error {
     readonly mismatches: string[];
     constructor(message: string, mismatches: string[]);
 }
+declare function defineEvalExpectations<const Suite extends EvalSuiteShape>(_suite: Suite, expectations: EvalExpectationsFor<NoInfer<Suite>>): EvalExpectationsFor<Suite>;
+declare function formatEvalResult(result: EvalSuiteResult<unknown, unknown, unknown>, options?: Omit<PrintEvalResultOptions, "output">): string;
 declare function printEvalResult(result: EvalSuiteResult<unknown, unknown, unknown>, options?: PrintEvalResultOptions): void;
 declare function evalExitCode(result: EvalSuiteResult<unknown, unknown, unknown>, expectations?: EvalExpectations): 0 | 1 | 2;
 declare function assertEvalTotals(result: EvalSuiteResult<unknown, unknown, unknown>, expected: EvalExpectedTotals): void;
 declare function assertEvalOutcomes(result: EvalSuiteResult<unknown, unknown, unknown>, expected: EvalExpectedOutcomes): void;
 declare function runEvalCli<Input, Output, Expected = unknown, const Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]>(options: RunEvalCliOptions<Input, Output, Expected, Metrics>): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;
 
+declare class EvalTimeoutError extends Error {
+    readonly timeoutMs: number;
+    constructor(timeoutMs: number);
+}
+declare class EvalAbortError extends Error {
+    constructor(message?: string);
+}
+
 declare function defineMetric<Input, Output, Score, Expected, const Name extends string = string>(metric: EvalMetric<Input, Output, Score, Expected, Name>): EvalMetric<Input, Output, Score, Expected, Name>;
 
 type ExactMatchOptions<Input, Output, Expected = unknown> = {
@@ -489,9 +613,16 @@ declare function contains<Input, Output, Expected = unknown, const Name extends
     expected: string | RegExp;
 }>;
 type NotContainsOptions<Input, Output, Expected = unknown> = ContainsOptions<Input, Output, Expected>;
-declare function notContains<Input, Output, Expected = unknown, const Name extends string = string>(options?: NotContainsOptions<Input, Output, Expected> & {
+declare function notContains<Input, Output, Expected = unknown, const Name extends string = string>(options: NotContainsOptions<Input, Output, Expected> & {
+    expected: Exclude<NotContainsOptions<Input, Output, Expected>["expected"], undefined>;
     name?: Name | undefined;
 }): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function notContains<Input, Output, Expected = unknown, const Name extends string = string>(options?: Omit<NotContainsOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: string | RegExp;
+}>;
 type ContainsListOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
     required?: boolean | undefined;
@@ -500,12 +631,26 @@ type ContainsListOptions<Input, Output, Expected = unknown> = {
 };
 type ContainsAllOptions<Input, Output, Expected = unknown> = ContainsListOptions<Input, Output, Expected>;
 declare function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(options: ContainsAllOptions<Input, Output, Expected> & {
+    expected: Exclude<ContainsAllOptions<Input, Output, Expected>["expected"], undefined>;
     name?: Name | undefined;
 }): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<ContainsAllOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: ReadonlyArray<string | RegExp>;
+}>;
 type ContainsAnyOptions<Input, Output, Expected = unknown> = ContainsListOptions<Input, Output, Expected>;
 declare function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(options: ContainsAnyOptions<Input, Output, Expected> & {
+    expected: Exclude<ContainsAnyOptions<Input, Output, Expected>["expected"], undefined>;
     name?: Name | undefined;
 }): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<ContainsAnyOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: ReadonlyArray<string | RegExp>;
+}>;
 type MatchesOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
     required?: boolean | undefined;
@@ -513,12 +658,26 @@ type MatchesOptions<Input, Output, Expected = unknown> = {
     expected?: SelectorOrValue<Input, Output, Expected, RegExp> | undefined;
 };
 declare function matches<Input, Output, Expected = unknown, const Name extends string = string>(options: MatchesOptions<Input, Output, Expected> & {
+    expected: Exclude<MatchesOptions<Input, Output, Expected>["expected"], undefined>;
     name?: Name | undefined;
 }): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function matches<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<MatchesOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: RegExp;
+}>;
 type DoesNotMatchOptions<Input, Output, Expected = unknown> = MatchesOptions<Input, Output, Expected>;
 declare function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(options: DoesNotMatchOptions<Input, Output, Expected> & {
+    expected: Exclude<DoesNotMatchOptions<Input, Output, Expected>["expected"], undefined>;
     name?: Name | undefined;
 }): EvalMetric<Input, Output, boolean, Expected, Name>;
+declare function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(options: Omit<DoesNotMatchOptions<Input, Output, Expected>, "expected"> & {
+    expected?: undefined;
+    name?: Name | undefined;
+}): EvalMetric<Input, Output, boolean, Expected, Name, {
+    expected: RegExp;
+}>;
 type MaxLengthOptions<Input, Output, Expected = unknown> = {
     name?: string | undefined;
     required?: boolean | undefined;
@@ -598,9 +757,15 @@ declare class EvalReporterDispatchError extends AggregateError {
     readonly phase: string;
     constructor(phase: string, errors: readonly unknown[]);
 }
+declare class EvalFailFastError extends Error {
+    readonly caseId: string;
+    readonly outcome: "fail" | "invalid";
+    constructor(caseId: string, outcome: "fail" | "invalid");
+}
 declare function runEvalSuite<Input, Output, Expected = unknown, const Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]>(options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;
 
 declare function selectPromptOutput(args: EvalMetricArgs<unknown, unknown, unknown>): string;
+declare function selectEvalCaseIds(result: EvalSuiteResult<unknown, unknown, unknown>, outcomes?: readonly EvalOutcomeStatus[]): string[];
 
 type EvalCaseLike = EvalCase<unknown, unknown>;
 type EvalCasesInput<Cases extends readonly EvalCaseLike[]> = Cases[number]["input"];
@@ -608,30 +773,9 @@ type EvalCaseExpected<Case> = Case extends {
     expected: infer Expected;
 } ? Expected : unknown;
 type EvalCasesExpected<Cases extends readonly EvalCaseLike[]> = EvalCaseExpected<Cases[number]>;
-type EvalMetricRequirements<Metric> = Metric extends EvalMetric<infer _Input, infer _Output, infer _Score, infer _Expected, infer _Name, infer Requirements> ? Requirements : Record<never, never>;
-type RequiredExpected<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
-    expected: unknown;
-}>;
-type RequiredContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
-    context: string[];
-}>;
-type RequiredRetrievalContext<Metrics extends readonly EvalMetric<never, never>[]> = Extract<EvalMetricRequirements<Metrics[number]>, {
-    retrievalContext: string[];
-}>;
-type EvalCaseFieldsForMetrics<Metrics extends readonly EvalMetric<never, never>[]> = ([
-    RequiredExpected<Metrics>
-] extends [never] ? Record<never, never> : {
-    expected: RequiredExpected<Metrics>["expected"];
-}) & ([RequiredContext<Metrics>] extends [never] ? Record<never, never> : {
-    context: string[];
-}) & ([RequiredRetrievalContext<Metrics>] extends [never] ? Record<never, never> : {
-    retrievalContext: string[];
-});
-type EvalCasesForMetrics<Cases extends readonly EvalCaseLike[], Metrics extends readonly EvalMetric<never, never>[]> = {
-    readonly [Index in keyof Cases]: Cases[Index] & EvalCaseFieldsForMetrics<Metrics>;
-};
-type DefinedEvalSuite<Cases extends readonly EvalCaseLike[], Output, Metrics extends readonly EvalMetric<EvalCasesInput<Cases>, Output, unknown, EvalCasesExpected<Cases>, string>[]> = Omit<RunEvalSuiteOptions<EvalCasesInput<Cases>, Output, EvalCasesExpected<Cases>, Metrics>, "cases" | "target" | "metrics"> & {
+type DefinedEvalSuite<Cases extends readonly EvalCaseLike[], Output, Metrics extends readonly EvalMetric<EvalCasesInput<Cases>, Output, unknown, EvalCasesExpected<Cases>, string>[]> = Omit<RunEvalSuiteOptions<EvalCasesInput<Cases>, Output, EvalCasesExpected<Cases>, Metrics>, "caseIds" | "cases" | "target" | "metrics"> & {
     cases: Cases & EvalCasesForMetrics<NoInfer<Cases>, NoInfer<Metrics>>;
+    caseIds?: readonly Cases[number]["id"][] | undefined;
     target: EvalTarget<EvalCasesInput<Cases>, Output, EvalCasesExpected<Cases>>;
     metrics: Metrics;
 };
@@ -662,6 +806,8 @@ type EvalMetricFactory<Input, Output, Expected = unknown> = {
     }): EvalMetric<Input, Output, Score, Expected, Name>;
 };
 declare function defineEvalCases<const Cases extends readonly EvalCaseLike[]>(cases: Cases): Cases;
+declare function createEvalTypes<Input, Output, Expected = unknown>(): EvalMetricFactory<Input, Output, Expected>;
+/** @deprecated Use createEvalTypes() when defining typed custom metrics. */
 declare function defineEvalSuite<Input, Output, Expected = unknown>(): EvalMetricFactory<Input, Output, Expected>;
 declare function defineEvalSuite<const Cases extends readonly EvalCaseLike[], const Target extends (input: EvalCasesInput<Cases>, testCase: EvalCase<EvalCasesInput<Cases>, EvalCasesExpected<Cases>>) => unknown, const Metrics extends readonly EvalMetric<EvalCasesInput<Cases>, Awaited<ReturnType<Target>>, unknown, EvalCasesExpected<Cases>, string>[]>(options: DefinedEvalSuite<Cases, Awaited<ReturnType<Target>>, Metrics> & {
     target: Target;
@@ -669,4 +815,4 @@ declare function defineEvalSuite<const Cases extends readonly EvalCaseLike[], co
     target: Target;
 };
 
-export { type AbstentionCategory, type AbstentionOptions, AgentEvalSuspensionError, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, EvalReporterDispatchError, type EvalReporterErrorPolicy, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };
+export { type AbstentionCategory, type AbstentionOptions, AgentEvalSuspensionError, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAbortError, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectationsFor, type EvalExpectedOutcomes, type EvalExpectedOutcomesFor, type EvalExpectedTotals, EvalFailFastError, type EvalInvalidKind, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalProgressEvent, type EvalRedactionContext, type EvalRedactor, type EvalReportArgs, type EvalReporter, EvalReporterDispatchError, type EvalReporterErrorPolicy, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalShard, type EvalSuiteResult, type EvalTarget, type EvalTargetContext, type EvalTargetUsageSelector, EvalTimeoutError, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, createEvalTypes, defaultEvalTraceSelector, defineEvalCases, defineEvalExpectations, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, formatEvalResult, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectEvalCaseIds, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };