@agtlantis/eval 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,2738 @@
+import { Provider, ProviderType, ProviderPricing } from '@agtlantis/core';
+export { ANTHROPIC_PRICING, CostResult, DEFAULT_PRICING_CONFIG, FilePromptRepositoryOptions, FileSource, FileSourceBase64, FileSourceData, FileSourceDisplayInfo, FileSourcePath, FileSourceUrl, FileSystem, FoundFileSource, GOOGLE_PRICING, ModelPricing, OPENAI_PRICING, PricingConfig, PromptRepository, PromptTemplate, ResolveOptions, calculateCostFromUsage, compileTemplate, createFilePromptRepository, getFileSourceDisplayInfo, getFileSourcesDisplayInfo, inferMediaType, isFileSource, isFileSourceBase64, isFileSourceData, isFileSourcePath, isFileSourceUrl, resolveFileSource, resolveFileSourcesInInput, scanForFileSources } from '@agtlantis/core';
+export { MockCall, MockProvider, mock } from '@agtlantis/core/testing';
+import { z } from 'zod';
+
+/**
+ * Simplified token usage type for eval package.
+ *
+ * This is a subset of AI SDK's LanguageModelUsage that only includes
+ * the properties eval actually tracks. The cost-helpers module handles
+ * conversion when calling @agtlantis/core's pricing calculator.
+ *
+ * @example
+ * ```typescript
+ * const usage: EvalTokenUsage = {
+ *   inputTokens: 100,
+ *   outputTokens: 50,
+ *   totalTokens: 150,
+ * }
+ * ```
+ */
+interface EvalTokenUsage {
+    /** Number of input (prompt) tokens */
+    inputTokens: number;
+    /** Number of output (completion) tokens */
+    outputTokens: number;
+    /** Total tokens (input + output) */
+    totalTokens: number;
+}
+/**
+ * Simplified agent configuration for evaluation.
+ * Only requires fields needed for eval purposes.
+ *
+ * For agents from `ai-agents` package with full AgentConfig,
+ * use `toEvalAgent()` adapter to convert them.
+ */
+interface EvalAgentConfig {
+    /** Agent name for identification */
+    name: string;
+    /** Agent description (used by Judge for context) */
+    description?: string;
+    /** Additional custom fields */
+    [key: string]: unknown;
+}
+/**
+ * Agent prompt template.
+ */
+interface AgentPrompt<TInput> {
+    /** Prompt unique ID for version tracking */
+    id: string;
+    /** Version string (e.g., "1.0.0") */
+    version: string;
+    /** System prompt */
+    system: string;
+    /** User template string (for serialization/history) */
+    userTemplate?: string;
+    /** User prompt builder function */
+    renderUserPrompt: (input: TInput) => string;
+    /** Additional custom fields */
+    [key: string]: unknown;
+}
+/**
+ * Base metadata type shared by all LLM-using components (Agent, Judge, Improver).
+ * Provides consistent structure for tracking token usage and model information.
+ *
+ * @example
+ * ```typescript
+ * const metadata: ComponentMetadata = {
+ *   tokenUsage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
+ *   model: 'gpt-4o',
+ * }
+ * ```
+ */
+interface ComponentMetadata {
+    /** Token usage from the LLM call (AI SDK LanguageModelUsage format) */
+    tokenUsage?: EvalTokenUsage;
+    /** Model identifier used for the LLM call */
+    model?: string;
+    /** Additional custom fields */
+    [key: string]: unknown;
+}
+/**
+ * Agent execution result metadata.
+ * Extends ComponentMetadata with agent-specific fields.
+ */
+interface AgentMetadata extends ComponentMetadata {
+    /** Prompt version used for execution */
+    promptVersion?: string;
+    /** Execution duration in milliseconds */
+    duration?: number;
+}
+/**
+ * Judge evaluation metadata.
+ * Tracks token usage and model for cost calculation.
+ */
+interface JudgeMetadata extends ComponentMetadata {
+}
+/**
+ * Improver analysis metadata.
+ * Tracks token usage and model for cost calculation.
+ */
+interface ImproverMetadata extends ComponentMetadata {
+}
+/**
+ * Agent execution result.
+ */
+interface AgentResult<TOutput> {
+    result: TOutput;
+    metadata?: AgentMetadata;
+}
+/**
+ * Simplified Agent interface for evaluation.
+ *
+ * @example
+ * ```typescript
+ * // Direct implementation
+ * const myAgent: EvalAgent<string, string> = {
+ *   config: { name: 'MyAgent', description: 'A simple agent' },
+ *   prompt: { id: 'prompt-1', version: '1.0.0', system: '...', renderUserPrompt: (input) => input },
+ *   execute: async (input) => ({ result: `Processed: ${input}` })
+ * }
+ *
+ * // Or adapt from full ai-agents Agent
+ * const evalAgent = toEvalAgent(fullAgent)
+ * ```
+ */
+interface EvalAgent<TInput, TOutput> {
+    readonly config: EvalAgentConfig;
+    readonly prompt: AgentPrompt<TInput>;
+    execute(input: TInput, options?: unknown): Promise<AgentResult<TOutput>>;
+}
+/**
+ * Full AgentConfig interface (compatible with ai-agents package).
+ * Used for type-safe adaptation.
+ */
+interface FullAgentConfig {
+    name: string;
+    role: 'generator' | 'analyzer' | 'validator' | 'enhancer';
+    streaming: 'required' | 'optional' | 'none';
+    execution: 'batch' | 'realtime';
+    conversation?: 'single-turn' | 'multi-turn';
+    description?: string;
+    [key: string]: unknown;
+}
+/**
+ * Full Agent interface (compatible with ai-agents package).
+ * Used for type-safe adaptation.
+ */
+interface FullAgent<TInput, TOutput> {
+    readonly config: FullAgentConfig;
+    readonly prompt: AgentPrompt<TInput>;
+    execute(input: TInput, options?: unknown): Promise<{
+        result: TOutput;
+        metadata: {
+            duration: number;
+            promptVersion: string;
+            tokenUsage?: EvalTokenUsage;
+            model?: string;
+            retryCount?: number;
+            traceId?: string;
+            [key: string]: unknown;
+        };
+    }>;
+}
+/**
+ * Adapts a full Agent (from ai-agents) to EvalAgent for evaluation.
+ * Extracts only the fields needed for evaluation.
+ *
+ * @example
+ * ```typescript
+ * import { scenarioGenerator } from './agents/mce'
+ *
+ * const evalAgent = toEvalAgent(scenarioGenerator)
+ * const suite = createEvalSuite({ agent: evalAgent, ... })
+ * ```
+ */
+declare function toEvalAgent<TInput, TOutput>(agent: FullAgent<TInput, TOutput>): EvalAgent<TInput, TOutput>;
+/**
+ * Metadata for file content.
+ */
+interface FileContentMetadata {
+    /** File size in bytes */
+    size?: number;
+    /** Full resolved path (for loaded files) */
+    fullPath?: string;
+    /** Whether the content was created inline (not from disk) */
+    inline?: boolean;
+    /** Additional custom metadata */
+    [key: string]: unknown;
+}
+interface FileContent {
+    /** File path (relative or absolute) - used as identifier */
+    path: string;
+    /** File content as string (text files only for Phase 5.3) */
+    content: string;
+    /** Optional MIME type hint (defaults to 'text/plain') */
+    mediaType?: string;
+    /** Optional encoding (defaults to 'utf-8') */
+    encoding?: BufferEncoding;
+    /** Optional metadata (e.g., original size, full path, etc.) */
+    metadata?: FileContentMetadata;
+}
+interface TestCase<TInput> {
+    id?: string;
+    input: TInput;
+    tags?: string[];
+    description?: string;
+    expectedOutput?: unknown;
+    /**
+     * Optional file context for agent and judge (Phase 5.3).
+     * Files are passed to Judge for evaluation context.
+     * For Agent access, include files in the input type directly.
+     *
+     * @deprecated Use FileSource in input directly for flexible file handling
+     */
+    files?: FileContent[];
+}
+interface MetricsResult {
+    latencyMs: number;
+    tokenUsage: EvalTokenUsage;
+}
+interface Criterion {
+    id: string;
+    name: string;
+    description: string;
+    weight?: number;
+}
+/**
+ * Zod error issue - minimal type compatible with ZodError.errors.
+ * Using `readonly` and rest index to be compatible with Zod's discriminated union.
+ */
+type ZodIssue = {
+    readonly code: string;
+    readonly path: readonly (string | number)[];
+    readonly message: string;
+};
+/**
+ * Result of programmatic schema validation.
+ */
+interface SchemaValidationResult {
+    /** Whether the output matches the schema */
+    valid: boolean;
+    /** Validation errors if invalid (Zod issue format) */
+    errors?: readonly ZodIssue[];
+    /** Human-readable error summary */
+    errorSummary?: string;
+}
+/**
+ * Validator function type for programmatic validation.
+ * Returns validation result with binary pass/fail outcome.
+ */
+type ValidatorFn = (output: unknown) => SchemaValidationResult;
+/**
+ * Extended criterion with optional programmatic validator.
+ * Validators run before LLM evaluation with binary scoring (0 or 100).
+ *
+ * @example
+ * ```typescript
+ * import { z } from 'zod'
+ * import { schema } from '@agtlantis/eval'
+ *
+ * const criterion = schema({
+ *   schema: z.object({ name: z.string() }),
+ *   weight: 2,
+ * })
+ * ```
+ */
+interface ValidatorCriterion extends Criterion {
+    /**
+     * Optional programmatic validator.
+     * If provided and fails, score is automatically 0.
+     * If provided and passes, score is automatically 100.
+     */
+    validator?: ValidatorFn;
+}
+interface Verdict {
+    criterionId: string;
+    score: number;
+    reasoning: string;
+    passed: boolean;
+}
+interface TestResult<TInput, TOutput> {
+    testCase: TestCase<TInput>;
+    output: TOutput;
+    metrics: MetricsResult;
+    error?: Error;
+}
+interface TestResultWithVerdict<TInput, TOutput> extends TestResult<TInput, TOutput> {
+    verdicts: Verdict[];
+    overallScore: number;
+    passed: boolean;
+    /** Judge metadata for cost tracking */
+    judgeMetadata?: JudgeMetadata;
+}
+/**
+ * Statistics from running the same test multiple times.
+ * Used to measure consistency and reliability of LLM-based agents.
+ */
+interface IterationStats {
+    /** Total number of iterations run */
+    iterations: number;
+    /** Score from each iteration */
+    scores: number[];
+    /** Average score across all iterations */
+    mean: number;
+    /** Standard deviation (lower = more consistent) */
+    stdDev: number;
+    /** Lowest score achieved */
+    min: number;
+    /** Highest score achieved */
+    max: number;
+    /** Pass rate as decimal (0-1, e.g., 0.67 = 67%) */
+    passRate: number;
+    /** Number of iterations that passed */
+    passCount: number;
+}
+/**
+ * Extended iteration statistics for multi-turn tests.
+ * Includes turn-count metrics and termination type distribution.
+ *
+ * @example
+ * ```typescript
+ * if (hasMultiTurnIterationData(result)) {
+ *   console.log(`Average turns: ${result.multiTurnIterationStats.avgTurns}`)
+ *   console.log(`Termination types: ${JSON.stringify(result.multiTurnIterationStats.terminationCounts)}`)
+ * }
+ * ```
+ */
+interface MultiTurnIterationStats extends IterationStats {
+    /** Average number of turns across all iterations */
+    avgTurns: number;
+    /** Minimum turns in any iteration */
+    minTurns: number;
+    /** Maximum turns in any iteration */
+    maxTurns: number;
+    /** Distribution of termination types across iterations (e.g., { condition: 2, maxTurns: 1 }) */
+    terminationCounts: Record<string, number>;
+}
+/**
+ * Discriminator for eval result types.
+ * Used for exhaustive pattern matching on result variants.
+ */
+type EvalResultKind = 'single-turn' | 'single-turn-iterated' | 'multi-turn' | 'multi-turn-iterated';
+/**
+ * Properties present when test ran with multiple iterations.
+ * Extracted as a separate interface for composition.
+ */
+interface IterationData<TInput, TOutput> {
+    /** Aggregated statistics across all iterations */
+    iterationStats: IterationStats;
+    /** Individual results from each iteration */
+    iterationResults: TestResultWithVerdict<TInput, TOutput>[];
+}
+/**
+ * Single conversation entry in multi-turn tests.
+ */
+interface ConversationEntry<TInput, TOutput> {
+    /** Turn number (1-based) */
+    turn: number;
+    /** Input provided for this turn */
+    input: TInput;
+    /** Output from agent (undefined if execution failed) */
+    output: TOutput | undefined;
+    /** Agent execution metadata */
+    metadata?: AgentMetadata;
+}
+/**
+ * Termination info for multi-turn tests.
+ * Compatible with TerminationCheckResult from multi-turn module.
+ */
+interface TerminationInfo {
+    /** Whether the conversation terminated */
+    terminated: boolean;
+    /** Human-readable reason for termination */
+    reason: string;
+    /** Type of termination (condition, maxTurns, error, exhausted) */
+    terminationType?: string;
+    /** The condition that caused termination (if applicable) */
+    matchedCondition?: unknown;
+}
+/**
+ * Properties present for multi-turn test results.
+ * Extracted as a separate interface for composition.
+ */
+interface MultiTurnData<TInput, TOutput> {
+    /** Full conversation history */
+    conversationHistory: ConversationEntry<TInput, TOutput>[];
+    /** Total turns executed */
+    totalTurns: number;
+    /** Human-readable termination reason */
+    terminationReason: string;
+    /** Full termination check result */
+    termination: TerminationInfo;
+}
+/**
+ * Single-turn test result with single iteration (base case).
+ * No iteration stats, no multi-turn data.
+ */
+interface SingleTurnResult<TInput, TOutput> extends TestResultWithVerdict<TInput, TOutput> {
+    readonly kind: 'single-turn';
+}
+/**
+ * Single-turn test result with multiple iterations.
+ * Has iteration stats but no multi-turn data.
+ */
+interface SingleTurnIteratedResult<TInput, TOutput> extends TestResultWithVerdict<TInput, TOutput>, IterationData<TInput, TOutput> {
+    readonly kind: 'single-turn-iterated';
+}
+/**
+ * Multi-turn test result with single iteration.
+ * Has multi-turn data but no iteration stats.
+ */
+interface MultiTurnResult<TInput, TOutput> extends TestResultWithVerdict<TInput, TOutput>, MultiTurnData<TInput, TOutput> {
+    readonly kind: 'multi-turn';
+}
+/**
+ * Multi-turn test result with multiple iterations.
+ * Has both multi-turn data and iteration stats.
+ */
+interface MultiTurnIteratedResult<TInput, TOutput> extends TestResultWithVerdict<TInput, TOutput>, IterationData<TInput, TOutput>, MultiTurnData<TInput, TOutput> {
+    readonly kind: 'multi-turn-iterated';
+    /** Multi-turn specific iteration statistics */
+    multiTurnIterationStats: MultiTurnIterationStats;
+}
+/**
+ * Unified eval result type - discriminated union of all result kinds.
+ *
+ * Use pattern matching on `kind` for exhaustive handling:
+ * @example
+ * ```typescript
+ * switch (result.kind) {
+ *   case 'single-turn':
+ *     // No iteration stats, no multi-turn data
+ *     break
+ *   case 'single-turn-iterated':
+ *     console.log(result.iterationStats.mean)  // Type-safe
+ *     break
+ *   case 'multi-turn':
+ *     console.log(result.conversationHistory)  // Type-safe
+ *     break
+ *   case 'multi-turn-iterated':
+ *     console.log(result.multiTurnIterationStats.avgTurns)  // Type-safe
+ *     break
+ * }
+ * ```
+ */
+type EvalTestResult<TInput, TOutput> = SingleTurnResult<TInput, TOutput> | SingleTurnIteratedResult<TInput, TOutput> | MultiTurnResult<TInput, TOutput> | MultiTurnIteratedResult<TInput, TOutput>;
+/**
+ * Check if result is from a single-turn test (either iterated or not).
+ *
+ * @example
+ * ```typescript
+ * if (isSingleTurnResult(result)) {
+ *   // result is SingleTurnResult | SingleTurnIteratedResult
+ *   console.log('Single turn test')
+ * }
+ * ```
+ */
+declare function isSingleTurnResult<TInput, TOutput>(result: EvalTestResult<TInput, TOutput>): result is SingleTurnResult<TInput, TOutput> | SingleTurnIteratedResult<TInput, TOutput>;
+/**
+ * Check if result is from a multi-turn test (either iterated or not).
+ *
+ * @example
+ * ```typescript
+ * if (isMultiTurnResult(result)) {
+ *   // result is MultiTurnResult | MultiTurnIteratedResult
+ *   console.log(`Turns: ${result.totalTurns}`)  // Type-safe
+ *   for (const entry of result.conversationHistory) {  // Type-safe
+ *     console.log(`Turn ${entry.turn}: ${entry.input}`)
+ *   }
+ * }
+ * ```
+ */
+declare function isMultiTurnResult<TInput, TOutput>(result: EvalTestResult<TInput, TOutput>): result is MultiTurnResult<TInput, TOutput> | MultiTurnIteratedResult<TInput, TOutput>;
+/**
+ * Check if result has iteration data (multiple iterations ran).
+ *
+ * @example
+ * ```typescript
+ * if (isIteratedResult(result)) {
+ *   // result is SingleTurnIteratedResult | MultiTurnIteratedResult
+ *   console.log(`Mean score: ${result.iterationStats.mean}`)  // Type-safe
+ *   console.log(`Pass rate: ${result.iterationStats.passRate}`)  // Type-safe
+ * }
+ * ```
+ */
+declare function isIteratedResult<TInput, TOutput>(result: EvalTestResult<TInput, TOutput>): result is SingleTurnIteratedResult<TInput, TOutput> | MultiTurnIteratedResult<TInput, TOutput>;
+
+/**
+ * Context passed to JudgePrompt.renderUserPrompt().
+ */
+interface JudgeContext {
+    agentDescription: string;
+    input: unknown;
+    output: unknown;
+    criteria: Criterion[];
+    files?: FileContent[];
+}
+/**
+ * Context for evaluating agent output.
+ *
+ * @example
+ * ```typescript
+ * const result = await judge.evaluate({
+ *   input: { query: 'Hello' },
+ *   output: { response: 'Hi there!' },
+ *   agentDescription: 'A friendly chatbot',
+ *   files: [{ path: 'context.md', content: '...' }],
+ * })
+ * ```
+ */
+interface EvalContext {
+    input: unknown;
+    output: unknown;
+    agentDescription: string;
+    files?: FileContent[];
+}
+interface JudgeResult {
+    verdicts: Verdict[];
+    overallScore: number;
+    passed: boolean;
+    metadata?: JudgeMetadata;
+}
+interface JudgePrompt {
+    id: string;
+    version: string;
+    system: string;
+    renderUserPrompt: (context: JudgeContext) => string;
+}
+interface JudgeConfig {
+    provider: Provider;
+    prompt?: JudgePrompt;
+    criteria: Criterion[];
+    passThreshold?: number;
+    /** Model name for cost tracking (e.g., 'gpt-4o', 'gemini-2.5-flash') */
+    model?: string;
+}
+/**
+ * LLM-as-Judge evaluator interface.
+ *
+ * @example
+ * ```typescript
+ * const judge = createJudge({ llm, prompt, criteria })
+ *
+ * const result = await judge.evaluate({
+ *   input: { query: 'What is 2+2?' },
+ *   output: { answer: '4' },
+ *   agentDescription: 'A math tutor agent',
+ *   files: [{ path: 'reference.md', content: '...' }],
+ * })
+ * ```
+ */
+interface Judge {
+    evaluate(context: EvalContext): Promise<JudgeResult>;
+}
+
+interface AggregatedMetrics {
+    avgLatencyMs: number;
+    totalTokens: number;
+    totalEstimatedCost?: number;
+}
+/**
+ * Pure data interface - use utility functions for operations.
+ *
+ * @example
+ * ```typescript
+ * for (const suggestion of report.suggestions) {
+ *   console.log(suggestionDiff(suggestion))
+ *   console.log(suggestionPreview(suggestion))
+ *   suggestion.approved = true
+ * }
+ *
+ * const newPrompt = applyPromptSuggestions(agent.prompt, report.suggestions)
+ * ```
+ */
+interface Suggestion {
+    type: 'system_prompt' | 'user_prompt' | 'parameters';
+    priority: 'high' | 'medium' | 'low';
+    currentValue: string;
+    suggestedValue: string;
+    reasoning: string;
+    expectedImprovement: string;
+    approved?: boolean;
+    modified?: boolean;
+}
+interface ImproveResult {
+    suggestions: Suggestion[];
+    metadata?: ImproverMetadata;
+}
+interface ImproverContext {
+    agentPrompt: AgentPrompt<any>;
+    evaluatedResults: EvalTestResult<any, any>[];
+    aggregatedMetrics: AggregatedMetrics;
+}
+interface ImproverPrompt {
+    id: string;
+    version: string;
+    system: string;
+    renderUserPrompt: (context: ImproverContext) => string;
+}
+interface ImproverConfig {
+    provider: Provider;
+    prompt?: ImproverPrompt;
+    /** Model name for cost tracking (e.g., 'gpt-4o', 'gemini-2.5-flash') */
+    model?: string;
+}
+interface Improver {
+    improve(agentPrompt: AgentPrompt<any>, results: EvalTestResult<any, any>[]): Promise<ImproveResult>;
+}
+
+/** Cost breakdown by component (Agent, Judge, Improver) */
+interface CostBreakdown {
+    agent?: number;
+    judge?: number;
+    improver?: number;
+    total?: number;
+}
+/** Cost summary aggregated across all test results */
+interface CostSummary {
+    total: number;
+    byComponent: {
+        agent: number;
+        judge: number;
+        improver?: number;
+    };
+}
+interface MetricsWithCost {
+    latencyMs: number;
+    tokenUsage: EvalTokenUsage;
+    costBreakdown: CostBreakdown;
+}
+/** Test result with cost breakdown, returned by addCostsToResults() */
+interface TestResultWithCost<TInput, TOutput> {
+    testCase: {
+        id?: string;
+        input: TInput;
+        tags?: string[];
+        description?: string;
+        expectedOutput?: unknown;
+    };
+    output: TOutput;
+    metrics: MetricsWithCost;
+    error?: Error;
+    verdicts: Array<{
+        criterionId: string;
+        score: number;
+        reasoning: string;
+        passed: boolean;
+    }>;
+    overallScore: number;
+    passed: boolean;
+}
+/** Pricing configuration for eval */
+interface EvalPricingConfig {
+    /** Provider-specific pricing overrides. Key is provider name (e.g., 'google', 'openai'), value is model pricing. */
+    providerPricing?: Partial<Record<ProviderType, ProviderPricing>>;
+}
+/** Minimal result interface compatible with TestResultWithVerdict and TestResultWithIteration */
+interface ResultForCostCalculation<TInput, TOutput> {
+    testCase: {
+        id?: string;
+        input: TInput;
+        tags?: string[];
+        description?: string;
+        expectedOutput?: unknown;
+    };
+    output: TOutput;
+    metrics: {
+        latencyMs: number;
+        tokenUsage: EvalTokenUsage;
+    };
+    error?: Error;
+    verdicts: Array<{
+        criterionId: string;
+        score: number;
+        reasoning: string;
+        passed: boolean;
+    }>;
+    overallScore: number;
+    passed: boolean;
+    agentMetadata?: {
+        tokenUsage?: EvalTokenUsage;
+        model?: string;
+        provider?: string;
+    };
+    judgeMetadata?: {
+        tokenUsage?: EvalTokenUsage;
+        model?: string;
+        provider?: string;
+    };
+}
+interface ReportForCostCalculation<TInput, TOutput> {
+    results: ResultForCostCalculation<TInput, TOutput>[];
+}
+declare function calculateResultCost<TInput, TOutput>(result: ResultForCostCalculation<TInput, TOutput>, config?: EvalPricingConfig): CostBreakdown;
+declare function calculateReportCosts<TInput, TOutput>(report: ReportForCostCalculation<TInput, TOutput>, config?: EvalPricingConfig): CostSummary;
+/** Add cost breakdown to each result. Returns new array (does not mutate original). */
+declare function addCostsToResults<TInput, TOutput>(results: ResultForCostCalculation<TInput, TOutput>[], config?: EvalPricingConfig): TestResultWithCost<TInput, TOutput>[];
+
+/**
+ * Reporter interface for saving/logging evaluation reports.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createJsonReporter('./reports')
+ * reporter.save(report, 'my-test')  // → ./reports/my-test-1736691234567.json
+ * ```
+ */
+interface Reporter<TInput = unknown, TOutput = unknown> {
+    /** Save report to file, returns file path (optional - not all reporters save files) */
+    save?(report: EvalReport<TInput, TOutput>, name: string): string;
+    /** Log report to console (optional) */
+    log?(report: EvalReport<TInput, TOutput>): void;
+}
+/**
+ * Common options for file-based reporters.
+ */
+interface FileReporterOptions {
+    /** Output directory (created if missing) */
+    outputDir: string;
+    /** Pricing config for cost calculation */
+    pricing?: EvalPricingConfig;
+    /** Add timestamp to filename (default: true) */
+    addTimestamp?: boolean;
+}
+/**
+ * Verbosity level for console output.
+ */
+type LogVerbosity = 'summary' | 'detailed' | 'full';
+/**
+ * Options for ConsoleReporter.
+ */
+interface ConsoleReporterOptions {
+    /** Verbosity level (default: 'summary') */
+    verbosity?: LogVerbosity;
+    /** Pricing config for cost display */
+    pricing?: EvalPricingConfig;
+}
+interface ReportSummary {
+    totalTests: number;
+    passed: number;
+    failed: number;
+    avgScore: number;
+    metrics: AggregatedMetrics;
+    /** Number of iterations run per test case (only present when iterations > 1) */
+    iterations?: number;
+    /** Average standard deviation across all tests */
+    avgStdDev?: number;
+    /** Average pass rate across all tests */
+    avgPassRate?: number;
+    /** Cost summary (set by CLI or manually via calculateReportCosts) */
+    costSummary?: CostSummary;
+}
+/**
+ * Evaluation report data.
+ * Pure data interface - use utility functions for operations.
+ *
+ * @example
+ * ```typescript
+ * const report = await suite.run(testCases)
+ *
+ * // Convert to markdown
+ * const markdown = reportToMarkdown(report)
+ *
+ * // Save to file
+ * await saveReportMarkdown(report, './reports/eval-report.md')
+ * ```
+ */
+interface EvalReport<TInput, TOutput> {
+    summary: ReportSummary;
+    /** Results - may include iteration stats when iterations > 1 */
+    results: EvalTestResult<TInput, TOutput>[];
+    suggestions: Suggestion[];
+    generatedAt: Date;
+    promptVersion: string;
+}
+/**
+ * Options for markdown report generation.
+ */
+interface ReportMarkdownOptions {
+    /** Include passed test details (default: false, collapsed) */
+    expandPassedTests?: boolean;
+    /** Include raw JSON output (default: false) */
+    includeRawOutput?: boolean;
+    /** Max length for output preview (default: 200) */
+    outputPreviewLength?: number;
+}
+/**
+ * Result of comparing two evaluation reports.
+ * Useful for tracking improvements across prompt versions.
+ *
+ * @example
+ * ```typescript
+ * const beforeReport = await suite.run(testCases)
+ * const afterReport = await suite.withAgent(improvedAgent).run(testCases)
+ * const comparison = compareReports(beforeReport, afterReport)
+ *
+ * console.log(`Score delta: ${comparison.scoreDelta}`)
+ * console.log(`Improved tests: ${comparison.improved.join(', ')}`)
+ * ```
+ */
+interface ReportComparison {
+    /** Change in average score (positive = improvement) */
+    scoreDelta: number;
+    /** Change in pass rate (positive = improvement) */
+    passRateDelta: number;
+    /** Changes in performance metrics */
+    metricsDelta: {
+        /** Change in average latency (ms) */
+        latencyMs: number;
+        /** Change in total token usage */
+        tokenUsage: number;
+    };
+    /** Test IDs that improved (score increased) */
+    improved: string[];
+    /** Test IDs that regressed (score decreased) */
+    regressed: string[];
+    /** Test IDs that were removed (in before but not in after) */
+    removed: string[];
+}
+
+/**
+ * Options for running test cases.
+ */
+interface RunOptions {
+    /** Maximum number of concurrent test case executions. Defaults to 1 (sequential). */
+    concurrency?: number;
+    /** Stop execution after the first test failure. Defaults to false. */
+    stopOnFirstFailure?: boolean;
+    /** AbortSignal for cancelling execution */
+    signal?: AbortSignal;
+    /**
+     * Number of times to run each test case. Defaults to 1.
+     * When > 1, results include iteration statistics (mean, stdDev, passRate).
+     */
+    iterations?: number;
+}
+/**
+ * Context required for executing a single test case.
+ * @internal
+ */
+interface ExecuteContext<TInput, TOutput> {
+    agent: EvalAgent<TInput, TOutput>;
+    judge: Judge;
+    agentDescription: string;
+}
+/**
+ * Executes a single test case and returns the result with verdict.
+ *
+ * Flow:
+ * 1. Execute agent with test input
+ * 2. Measure execution latency
+ * 3. Collect token usage from agent metadata
+ * 4. Evaluate output using Judge
+ * 5. Return combined result with verdicts
+ *
+ * @example
+ * ```typescript
+ * const result = await executeTestCase(
+ *   { id: 'test-1', input: { query: 'Hello' } },
+ *   { agent: myAgent, judge: myJudge, agentDescription: 'A friendly bot' }
+ * )
+ *
+ * console.log(result.passed)       // true/false
+ * console.log(result.overallScore) // 0-100
+ * console.log(result.verdicts)     // Verdict[]
+ * ```
+ */
+declare function executeTestCase<TInput, TOutput>(testCase: TestCase<TInput>, context: ExecuteContext<TInput, TOutput>, signal?: AbortSignal): Promise<SingleTurnResult<TInput, TOutput>>;
+/**
+ * Runs multiple test cases with configurable concurrency.
+ *
+ * Features:
+ * - Parallel execution with concurrency limit
+ * - Stop on first failure option
+ * - AbortSignal support for cancellation
+ *
+ * @example
+ * ```typescript
+ * const results = await runWithConcurrency(
+ *   testCases,
+ *   { agent: myAgent, judge: myJudge, agentDescription: 'Test agent' },
+ *   { concurrency: 5, stopOnFirstFailure: false }
+ * )
+ *
+ * console.log(`Passed: ${results.filter(r => r.passed).length}`)
+ * console.log(`Failed: ${results.filter(r => !r.passed).length}`)
+ * ```
+ */
+declare function runWithConcurrency<TInput, TOutput>(testCases: TestCase<TInput>[], context: ExecuteContext<TInput, TOutput>, options?: RunOptions): Promise<EvalTestResult<TInput, TOutput>[]>;
+
+/**
+ * Configuration for creating an EvalSuite.
+ *
+ * @example
+ * ```typescript
+ * const suite = createEvalSuite({
+ *   agent: myAgent,
+ *   judge: myJudge,
+ *   agentDescription: 'Recommends career paths based on student profiles',
+ * })
+ * ```
+ */
+interface EvalSuiteConfig<TInput, TOutput> {
+    /** The agent to evaluate */
+    agent: EvalAgent<TInput, TOutput>;
+    /** Human-readable description of what the agent does (used by Judge) */
+    agentDescription?: string;
+    /** Judge instance for evaluating agent outputs */
+    judge: Judge;
+    /** Improver instance for generating prompt improvement suggestions (optional) */
+    improver?: Improver;
+}
+/**
+ * Evaluation suite for running test cases against an agent.
+ *
+ * @example
+ * ```typescript
+ * const report = await suite.run(testCases, { concurrency: 3 })
+ * console.log(reportToMarkdown(report))
+ *
+ * // Test with a different agent
+ * const newReport = await suite.withAgent(improvedAgent).run(testCases)
+ * ```
+ */
+interface EvalSuite<TInput, TOutput> {
+    /**
+     * Run test cases and generate an evaluation report.
+     *
+     * @param testCases - Test cases to run
+     * @param options - Run options (concurrency, stopOnFirstFailure, signal)
+     * @returns Evaluation report with results, summary, and suggestions
+     */
+    run(testCases: TestCase<TInput>[], options?: RunOptions): Promise<EvalReport<TInput, TOutput>>;
+    /**
+     * Create a new suite with a different agent.
+     * Useful for A/B testing or testing prompt improvements.
+     *
+     * @param agent - New agent to use
+     * @returns New EvalSuite instance with the updated agent
+     */
+    withAgent(agent: EvalAgent<TInput, TOutput>): EvalSuite<TInput, TOutput>;
+}
+/**
+ * Create an evaluation suite for testing an agent.
+ *
+ * The suite orchestrates test execution, evaluation, and optional
+ * prompt improvement suggestions.
+ *
+ * @example
+ * ```typescript
+ * const suite = createEvalSuite({
+ *   agent: scenarioGenerator,
+ *   agentDescription: 'Recommends majors based on student profiles',
+ *   judge: createJudge({
+ *     llm: openaiClient,
+ *     prompt: defaultJudgePrompt,
+ *     criteria: [accuracy(), relevance()],
+ *   }),
+ * })
+ *
+ * const report = await suite.run(testCases, { concurrency: 3 })
+ * ```
+ */
+declare function createEvalSuite<TInput, TOutput>(config: EvalSuiteConfig<TInput, TOutput>): EvalSuite<TInput, TOutput>;
+
+/**
+ * Iteration statistics utilities for repeated test execution.
+ *
+ * These functions aggregate results from running the same test multiple times,
+ * providing statistical metrics like mean, standard deviation, and pass rate.
+ */
+
+/**
+ * Calculate iteration statistics from multiple test results.
+ *
+ * @param results - Results from running the same test multiple times
+ * @returns Aggregated statistics including mean, stdDev, and passRate
+ *
+ * @example
+ * ```typescript
+ * const stats = calculateIterationStats([
+ *   { overallScore: 85, passed: true, ... },
+ *   { overallScore: 90, passed: true, ... },
+ *   { overallScore: 80, passed: true, ... },
+ * ])
+ * // stats.mean = 85
+ * // stats.stdDev ≈ 4.08
+ * // stats.passRate = 1.0
+ * ```
+ */
+declare function calculateIterationStats(results: TestResultWithVerdict<unknown, unknown>[]): IterationStats;
+/**
+ * Calculate multi-turn specific iteration statistics.
+ *
+ * Extends base iteration stats with turn counts and termination type distribution.
+ * Used when aggregating multiple iterations of multi-turn tests.
+ *
+ * @param results - Results from running the same multi-turn test multiple times
+ * @returns Extended statistics including avgTurns, min/max turns, and terminationCounts
+ *
+ * @example
+ * ```typescript
+ * const stats = calculateMultiTurnIterationStats(results)
+ * // stats.avgTurns = 4.2
+ * // stats.minTurns = 3
+ * // stats.maxTurns = 6
+ * // stats.terminationCounts = { condition: 2, maxTurns: 1 }
+ * ```
+ */
+declare function calculateMultiTurnIterationStats<TInput, TOutput>(results: (MultiTurnResult<TInput, TOutput> | MultiTurnIteratedResult<TInput, TOutput>)[]): MultiTurnIterationStats;
+/**
+ * Select the result closest to the mean score.
+ * Used to pick a "representative" result for displaying verdicts/reasoning.
+ *
+ * The function preserves the full type of the input array, so if you pass
+ * `TestResultWithIteration[]`, you get back `TestResultWithIteration`.
+ *
+ * @param results - Array of results to choose from (must not be empty)
+ * @param mean - The mean score to compare against
+ * @returns The result with overallScore closest to mean
+ * @throws Error if results array is empty
+ */
+declare function selectRepresentativeResult<TInput, TOutput, T extends TestResultWithVerdict<TInput, TOutput> = TestResultWithVerdict<TInput, TOutput>>(results: T[], mean: number): T;
+/**
+ * Aggregate results from multiple iteration runs into iterated result types.
+ *
+ * Takes N arrays of results (one per iteration) and groups them by test case,
+ * calculating iteration statistics for each test case.
+ *
+ * For multi-turn tests, returns MultiTurnIteratedResult with multi-turn specific
+ * statistics like average turns, min/max turns, and termination type distribution.
+ *
+ * For single-turn tests, returns SingleTurnIteratedResult with base iteration stats.
+ *
+ * @param allIterationResults - Array of arrays: outer = iterations, inner = test cases
+ * @returns Aggregated results with iteration statistics
+ *
+ * @example
+ * ```typescript
+ * // 3 iterations, 2 test cases each
+ * const allResults = [
+ *   [testCase1_iter1, testCase2_iter1],  // iteration 1
+ *   [testCase1_iter2, testCase2_iter2],  // iteration 2
+ *   [testCase1_iter3, testCase2_iter3],  // iteration 3
+ * ]
+ *
+ * const aggregated = aggregateIterationResults(allResults)
+ * // aggregated[0] = testCase1 with stats from iter1, iter2, iter3
+ * // aggregated[1] = testCase2 with stats from iter1, iter2, iter3
+ *
+ * // For multi-turn tests:
+ * // aggregated[0].kind === 'multi-turn-iterated'
+ * // aggregated[0].multiTurnIterationStats = { avgTurns, minTurns, maxTurns, terminationCounts }
+ * ```
+ */
+declare function aggregateIterationResults<TInput, TOutput>(allIterationResults: EvalTestResult<TInput, TOutput>[][]): (SingleTurnIteratedResult<TInput, TOutput> | MultiTurnIteratedResult<TInput, TOutput>)[];
+/**
+ * Calculate average standard deviation across multiple test results.
+ * Used for report summary.
+ *
+ * @param results - Eval results (only iterated results have stats)
+ * @returns Average stdDev across all iterated tests, or undefined if no iteration data
+ */
+declare function calculateAvgStdDev<TInput, TOutput>(results: EvalTestResult<TInput, TOutput>[]): number | undefined;
+/**
+ * Calculate average pass rate across multiple test results.
+ * Used for report summary.
+ *
+ * @param results - Eval results (only iterated results have stats)
+ * @returns Average passRate across all iterated tests, or undefined if no iteration data
+ */
+declare function calculateAvgPassRate<TInput, TOutput>(results: EvalTestResult<TInput, TOutput>[]): number | undefined;
+
+/**
+ * Error codes for agent-eval operations
+ */
+declare enum EvalErrorCode {
+    LLM_API_ERROR = "LLM_API_ERROR",
+    LLM_RATE_LIMIT = "LLM_RATE_LIMIT",
+    LLM_TIMEOUT = "LLM_TIMEOUT",
+    JSON_PARSE_ERROR = "JSON_PARSE_ERROR",
+    VERDICT_PARSE_ERROR = "VERDICT_PARSE_ERROR",
+    TEMPLATE_COMPILE_ERROR = "TEMPLATE_COMPILE_ERROR",
+    AGENT_EXECUTION_ERROR = "AGENT_EXECUTION_ERROR",
+    INVALID_CONFIG = "INVALID_CONFIG",
+    MISSING_API_KEY = "MISSING_API_KEY",
+    PROMPT_NOT_FOUND = "PROMPT_NOT_FOUND",
+    PROMPT_INVALID_FORMAT = "PROMPT_INVALID_FORMAT",
+    PROMPT_WRITE_ERROR = "PROMPT_WRITE_ERROR",
+    PROMPT_READ_ERROR = "PROMPT_READ_ERROR",
+    SUGGESTION_APPLY_ERROR = "SUGGESTION_APPLY_ERROR",
+    SCHEMA_VALIDATION_ERROR = "SCHEMA_VALIDATION_ERROR",
+    SCHEMA_GENERATION_ERROR = "SCHEMA_GENERATION_ERROR",
+    FILE_READ_ERROR = "FILE_READ_ERROR",
+    FILE_WRITE_ERROR = "FILE_WRITE_ERROR",
+    FILE_TOO_LARGE = "FILE_TOO_LARGE",
+    CONCURRENT_MODIFICATION = "CONCURRENT_MODIFICATION",
+    UNKNOWN_ERROR = "UNKNOWN_ERROR"
+}
+interface EvalErrorOptions {
+    code: EvalErrorCode;
+    cause?: Error;
+    context?: Record<string, unknown>;
+}
+/**
+ * Custom error class for agent-eval operations.
+ * Provides structured error information including error code and optional context.
+ */
+declare class EvalError extends Error {
+    readonly code: EvalErrorCode;
+    readonly cause?: Error;
+    readonly context?: Record<string, unknown>;
+    constructor(message: string, options: EvalErrorOptions);
+    /**
+     * Creates an EvalError from an unknown error with a specific code.
+     */
+    static from(error: unknown, code: EvalErrorCode, context?: Record<string, unknown>): EvalError;
+    toJSON(): Record<string, unknown>;
+}
+
+/**
+ * Creates an LLM-as-Judge evaluator.
+ *
+ * @example
+ * ```typescript
+ * import { createJudge, defaultJudgePrompt, accuracy, consistency } from 'agent-eval'
+ * import { createGoogleProvider } from '@agtlantis/core'
+ *
+ * const provider = createGoogleProvider({ apiKey }).withDefaultModel('gemini-2.5-flash')
+ *
+ * const judge = createJudge({
+ *   provider,
+ *   prompt: defaultJudgePrompt,
+ *   criteria: [accuracy(), consistency()],
+ *   passThreshold: 70,
+ * })
+ *
+ * const result = await judge.evaluate({
+ *   input: { query: 'What is 2+2?' },
+ *   output: { answer: '4' },
+ *   agentDescription: 'A math tutor agent',
+ *   files: [{ path: 'reference.md', content: '...' }],
+ * })
+ *
+ * console.log(result.overallScore) // e.g., 85
+ * console.log(result.passed)       // true
+ * ```
+ */
+declare function createJudge(config: JudgeConfig): Judge;
+
+interface SchemaOptions<T> extends CriterionOptions {
+    schema: z.ZodType<T>;
+    /** Use unique IDs when using multiple validators */
+    id?: string;
+    name?: string;
+    description?: string;
+}
+/**
+ * Creates a schema validation criterion using Zod.
+ *
+ * Performs PROGRAMMATIC validation (not LLM-based).
+ * Scoring is binary: 100 if validation passes, 0 if it fails.
+ *
+ * @example
+ * ```typescript
+ * import { z } from 'zod'
+ * import { schema, createJudge, accuracy, defaultJudgePrompt } from '@agtlantis/eval'
+ *
+ * const RecipeSchema = z.object({
+ *   name: z.string(),
+ *   ingredients: z.array(z.object({
+ *     name: z.string(),
+ *     amount: z.string(),
+ *   })),
+ *   steps: z.array(z.string()).min(1),
+ * })
+ *
+ * const judge = createJudge({
+ *   llm: openaiClient,
+ *   prompt: defaultJudgePrompt,
+ *   criteria: [
+ *     schema({ schema: RecipeSchema, weight: 2 }),
+ *     accuracy(),
+ *   ],
+ * })
+ * ```
+ */
+declare function schema<T>(options: SchemaOptions<T>): ValidatorCriterion;
+
+interface CriterionOptions {
+    weight?: number;
+}
+/**
+ * Evaluates whether the agent's output is factually accurate
+ * and free from errors or hallucinations.
+ */
+declare function accuracy(options?: CriterionOptions): Criterion;
+/**
+ * Evaluates whether the agent's output is internally consistent
+ * and doesn't contradict itself or the provided context.
+ */
+declare function consistency(options?: CriterionOptions): Criterion;
+/**
+ * Evaluates whether the agent's output is relevant to the input
+ * and addresses the user's needs appropriately.
+ */
+declare function relevance(options?: CriterionOptions): Criterion;
+
+/**
+ * Converts an evaluation report to Markdown format.
+ *
+ * @example
+ * ```typescript
+ * const report = await suite.run(testCases)
+ * const markdown = reportToMarkdown(report)
+ * console.log(markdown)
+ * ```
+ */
+declare function reportToMarkdown<TInput, TOutput>(report: EvalReport<TInput, TOutput>, options?: ReportMarkdownOptions): string;
+/**
+ * Saves an evaluation report as a Markdown file.
+ *
+ * @example
+ * ```typescript
+ * const report = await suite.run(testCases)
+ * await saveReportMarkdown(report, './reports/eval-2024-01.md')
+ * ```
+ */
+declare function saveReportMarkdown<TInput, TOutput>(report: EvalReport<TInput, TOutput>, path: string, options?: ReportMarkdownOptions): Promise<void>;
+/**
+ * Compares two evaluation reports and returns the differences.
+ * Useful for tracking improvements across prompt versions.
+ *
+ * @example
+ * ```typescript
+ * const beforeReport = await suite.run(testCases)
+ * // ... apply improvements ...
+ * const afterReport = await suite.withAgent(improvedAgent).run(testCases)
+ *
+ * const comparison = compareReports(beforeReport, afterReport)
+ * console.log(`Score improved by ${comparison.scoreDelta} points`)
+ * console.log(`Tests improved: ${comparison.improved.join(', ')}`)
+ * console.log(`Tests regressed: ${comparison.regressed.join(', ')}`)
+ * ```
+ */
+declare function compareReports<TInput, TOutput>(before: EvalReport<TInput, TOutput>, after: EvalReport<TInput, TOutput>): ReportComparison;
+
+/**
+ * Reporter that saves EvalReport as JSON.
+ *
+ * @example
+ * ```typescript
+ * const reporter = new JsonReporter({ outputDir: './reports' })
+ * reporter.save(report, 'my-test')  // -> ./reports/my-test-1736691234567.json
+ *
+ * // Without timestamp
+ * const fixedReporter = new JsonReporter({
+ *   outputDir: './reports',
+ *   addTimestamp: false,
+ * })
+ * fixedReporter.save(report, 'round-1')  // -> ./reports/round-1.json
+ * ```
+ */
+declare class JsonReporter<TInput = unknown, TOutput = unknown> implements Reporter<TInput, TOutput> {
+    private readonly outputDir;
+    private readonly pricing?;
+    private readonly addTimestamp;
+    constructor(options: FileReporterOptions);
+    save(report: EvalReport<TInput, TOutput>, name: string): string;
+}
+
+interface MarkdownReporterOptions extends FileReporterOptions {
+    /** Markdown generation options */
+    markdown?: ReportMarkdownOptions;
+}
+/**
+ * Reporter that saves EvalReport as Markdown.
+ *
+ * @example
+ * ```typescript
+ * const reporter = new MarkdownReporter({ outputDir: './reports' })
+ * reporter.save(report, 'my-test')  // -> ./reports/my-test-1736691234567.md
+ *
+ * // With expanded passed tests
+ * const detailedReporter = new MarkdownReporter({
+ *   outputDir: './reports',
+ *   markdown: { expandPassedTests: true },
+ * })
+ * ```
+ */
+declare class MarkdownReporter<TInput = unknown, TOutput = unknown> implements Reporter<TInput, TOutput> {
+    private readonly outputDir;
+    private readonly addTimestamp;
+    private readonly markdownOptions;
+    constructor(options: MarkdownReporterOptions);
+    save(report: EvalReport<TInput, TOutput>, name: string): string;
+}
+
+/**
+ * Reporter that logs EvalReport to console.
+ *
+ * @example
+ * ```typescript
+ * const reporter = new ConsoleReporter({ verbosity: 'detailed' })
+ * reporter.log(report)  // Logs to console
+ *
+ * // With cost display
+ * const costReporter = new ConsoleReporter({
+ *   verbosity: 'summary',
+ *   pricing: GOOGLE_PRICING,
+ * })
+ * ```
+ */
+declare class ConsoleReporter<TInput = unknown, TOutput = unknown> implements Reporter<TInput, TOutput> {
+    private readonly verbosity;
+    private readonly pricing?;
+    constructor(options?: ConsoleReporterOptions);
+    log(report: EvalReport<TInput, TOutput>): void;
+    private logCostIfAvailable;
+}
+
+/**
+ * Combines multiple reporters to save/log to multiple outputs.
+ *
+ * @example
+ * ```typescript
+ * const reporter = new CompositeReporter([
+ *   new JsonReporter({ outputDir: './reports' }),
+ *   new ConsoleReporter({ verbosity: 'detailed' }),
+ * ])
+ * reporter.save(report, 'my-test')  // JSON 저장 + 콘솔 출력
+ * ```
+ */
+declare class CompositeReporter<TInput = unknown, TOutput = unknown> implements Reporter<TInput, TOutput> {
+    private readonly reporters;
+    constructor(reporters: Reporter<TInput, TOutput>[]);
+    /**
+     * Saves to all reporters that support saving.
+     * Returns the first successful file path (usually JsonReporter).
+     */
+    save(report: EvalReport<TInput, TOutput>, name: string): string;
+    log(report: EvalReport<TInput, TOutput>): void;
+}
+
+/**
+ * Create a JSON reporter.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createJsonReporter('./reports')
+ * reporter.save(report, 'my-test')  // → ./reports/my-test-1736691234567.json
+ * ```
+ */
+declare function createJsonReporter<TInput = unknown, TOutput = unknown>(outputDir: string, options?: Omit<FileReporterOptions, 'outputDir'>): JsonReporter<TInput, TOutput>;
+/**
+ * Create a Markdown reporter.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createMarkdownReporter('./reports')
+ * reporter.save(report, 'my-test')  // → ./reports/my-test-1736691234567.md
+ * ```
+ */
+declare function createMarkdownReporter<TInput = unknown, TOutput = unknown>(outputDir: string, options?: Omit<MarkdownReporterOptions, 'outputDir'>): MarkdownReporter<TInput, TOutput>;
+/**
+ * Create a console reporter.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createConsoleReporter({ verbosity: 'detailed' })
+ * reporter.log(report)  // Logs to console
+ * ```
+ */
+declare function createConsoleReporter<TInput = unknown, TOutput = unknown>(options?: ConsoleReporterOptions): ConsoleReporter<TInput, TOutput>;
+/**
+ * Create a composite reporter from multiple reporters.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createCompositeReporter([
+ *   createJsonReporter('./reports'),
+ *   createConsoleReporter({ verbosity: 'summary' }),
+ * ])
+ * ```
+ */
+declare function createCompositeReporter<TInput = unknown, TOutput = unknown>(reporters: Reporter<TInput, TOutput>[]): CompositeReporter<TInput, TOutput>;
+/**
+ * Convenience: Create JSON + Console reporter combo.
+ *
+ * @example
+ * ```typescript
+ * const reporter = createDefaultReporter('./reports', {
+ *   pricing: GOOGLE_PRICING,
+ *   verbosity: 'summary',
+ * })
+ * reporter.save(report, 'my-test')  // JSON 저장 + 콘솔 출력
+ * ```
+ */
+declare function createDefaultReporter<TInput = unknown, TOutput = unknown>(outputDir: string, options?: {
+    pricing?: EvalPricingConfig;
+    verbosity?: LogVerbosity;
+    addTimestamp?: boolean;
+}): CompositeReporter<TInput, TOutput>;
+
+/**
+ * Options for creating a report runner.
+ */
+interface ReportRunnerOptions {
+    /** Directory where reports will be saved */
+    outputDir: string;
+    /** Pricing config for cost calculation */
+    pricing?: EvalPricingConfig;
+    /** Verbosity level for console output (false to disable logging) */
+    verbosity?: LogVerbosity | false;
+}
+/**
+ * Result returned by the report runner.
+ */
+interface ReportRunnerResult<TInput, TOutput> {
+    /** The generated evaluation report */
+    report: EvalReport<TInput, TOutput>;
+    /** Path where the report was saved */
+    savedPath: string;
+}
+/**
+ * Creates a runner that automatically logs and saves reports.
+ *
+ * @param options - Runner configuration
+ * @returns A function that runs the suite and handles reporting
+ *
+ * @example
+ * ```typescript
+ * import { createReportRunner, GOOGLE_PRICING } from '@agtlantis/eval'
+ *
+ * const run = createReportRunner({
+ *   outputDir: './reports',
+ *   pricing: GOOGLE_PRICING,
+ *   verbosity: 'detailed',
+ * })
+ *
+ * const { report, savedPath } = await run(suite, testCases, 'my-evaluation')
+ * // Logs to console and saves to ./reports/my-evaluation-{timestamp}.json
+ * console.log(`Saved to: ${savedPath}`)
+ * ```
+ */
+declare function createReportRunner(options: ReportRunnerOptions): <TInput, TOutput>(suite: EvalSuite<TInput, TOutput>, testCases: TestCase<TInput>[], name: string) => Promise<ReportRunnerResult<TInput, TOutput>>;
+
+/** Storage abstraction for testability - allows injecting mock storage */
+interface HistoryStorage {
+    readFile: (path: string) => Promise<string>;
+    writeFile: (path: string, content: string) => Promise<void>;
+    exists: (path: string) => boolean;
+    mkdir: (path: string, options?: {
+        recursive?: boolean;
+    }) => Promise<string | undefined | void>;
+}
+declare const defaultHistoryStorage: HistoryStorage;
+interface ImprovementSession {
+    readonly sessionId: string;
+    readonly history: Readonly<ImprovementHistory>;
+    readonly canSave: boolean;
+    addRound(roundResult: RoundResult, updatedPrompt: SerializedPrompt): void;
+    complete(terminationReason: string): void;
+    save(): Promise<void>;
+    flush(): Promise<void>;
+}
+interface SessionConfig {
+    path?: string;
+    autoSave?: boolean;
+    storage?: HistoryStorage;
+    onAutoSaveError?: (error: Error) => void;
+}
+/** @throws EvalError with PROMPT_INVALID_FORMAT if userTemplate is missing */
+declare function serializePrompt<TInput>(prompt: AgentPrompt<TInput>): SerializedPrompt;
+/** Reconstructs renderUserPrompt using compileTemplate. */
+declare function deserializePrompt<TInput>(serialized: SerializedPrompt): AgentPrompt<TInput>;
+/** @throws EvalError with PROMPT_INVALID_FORMAT if prompt lacks userTemplate */
+declare function createSession<TInput>(initialPrompt: AgentPrompt<TInput>, config?: SessionConfig): ImprovementSession;
+/** Resume from a history file. Clears completion status to allow adding new rounds. */
+declare function resumeSession(path: string, config?: Omit<SessionConfig, 'path'>): Promise<ImprovementSession>;
+/** Save history to JSON file. Creates parent directories if needed. */
+declare function saveHistory(history: ImprovementHistory, path: string, storage?: HistoryStorage): Promise<void>;
+declare function loadHistory(path: string, storage?: HistoryStorage): Promise<ImprovementHistory>;
+
+/** Terminate when average score reaches threshold */
+interface TargetScoreCondition {
+    type: 'targetScore';
+    /** Score threshold (0-100) */
+    threshold: number;
+}
+/** Terminate after N rounds */
+interface MaxRoundsCondition {
+    type: 'maxRounds';
+    /** Maximum number of improvement rounds */
+    count: number;
+}
+/** Terminate when score doesn't improve for N consecutive rounds */
+interface NoImprovementCondition {
+    type: 'noImprovement';
+    /** Number of consecutive rounds without improvement */
+    consecutiveRounds: number;
+    /** Minimum score delta to count as improvement (default: 0) */
+    minDelta?: number;
+}
+/** Terminate when total cost exceeds budget */
+interface MaxCostCondition {
+    type: 'maxCost';
+    /** Maximum cost in USD */
+    maxUSD: number;
+}
+/** Custom condition with user-defined check function */
+interface CustomCycleCondition {
+    type: 'custom';
+    /** Function to check if termination condition is met */
+    check: (ctx: CycleContext) => boolean | Promise<boolean>;
+    /** Human-readable description (for debugging/logging) */
+    description?: string;
+}
+/** Discriminated union of termination conditions. Uses OR semantics - first match triggers. */
+type CycleTerminationCondition = TargetScoreCondition | MaxRoundsCondition | NoImprovementCondition | MaxCostCondition | CustomCycleCondition;
+/** Context available to termination condition checks */
+interface CycleContext {
+    /** Current round number (1-indexed) */
+    currentRound: number;
+    /** Average score from the latest round */
+    latestScore: number;
+    /** Score history from all previous rounds */
+    previousScores: number[];
+    /** Total accumulated cost in USD */
+    totalCost: number;
+    /** Full history of completed rounds */
+    history: RoundResult[];
+}
+/** Result when cycle should continue (no termination) */
+interface CycleContinueResult {
+    terminated: false;
+    reason: string;
+    /** Not present when not terminated (for type safety with discriminated union) */
+    matchedCondition?: never;
+}
+/** Result when cycle should terminate */
+interface CycleTerminatedResult {
+    terminated: true;
+    matchedCondition: CycleTerminationCondition;
+    reason: string;
+}
+type CycleTerminationResult = CycleContinueResult | CycleTerminatedResult;
+/**
+ * Data yielded after each improvement round for Human-in-the-Loop (HITL) control.
+ * The AsyncGenerator yields this after each round, allowing inspection and decision.
+ */
+interface RoundYield {
+    /** Result of the completed round */
+    roundResult: RoundResult;
+    /** Current cycle context */
+    context: CycleContext;
+    /** Suggestions awaiting approval */
+    pendingSuggestions: Suggestion[];
+    /** Termination check result (use isCycleTerminated() to check if terminated) */
+    terminationCheck: CycleTerminationResult;
+}
+/** Decision from the caller after reviewing a round */
+interface RoundDecision {
+    /** Action to take */
+    action: 'continue' | 'stop' | 'rollback';
+    /** Target round for rollback (required if action is 'rollback') */
+    rollbackToRound?: number;
+    /** Suggestions approved by user (optional override) */
+    approvedSuggestions?: Suggestion[];
+}
+/** Cost breakdown for a single round */
+interface RoundCost {
+    /** Agent LLM cost in USD */
+    agent: number;
+    /** Judge LLM cost in USD */
+    judge: number;
+    /** Improver LLM cost in USD */
+    improver: number;
+    /** Total cost in USD */
+    total: number;
+}
+/** Result of a single improvement round */
+interface RoundResult {
+    /** Round number (1-indexed) */
+    round: number;
+    /** When this round completed */
+    completedAt: Date;
+    /** Full evaluation report */
+    report: EvalReport<unknown, unknown>;
+    /** All suggestions generated by improver */
+    suggestionsGenerated: Suggestion[];
+    /** Suggestions that were approved/applied */
+    suggestionsApproved: Suggestion[];
+    /** Prompt snapshot at start of this round (for rollback) */
+    promptSnapshot: SerializedPrompt;
+    /** Prompt version after applying suggestions */
+    promptVersionAfter: string;
+    /** Cost breakdown for this round */
+    cost: RoundCost;
+    /** Score change from previous round (null for first round) */
+    scoreDelta: number | null;
+}
+/**
+ * Serialized prompt for JSON storage.
+ * Note: renderUserPrompt cannot be serialized; use compileTemplate(userTemplate) to reconstruct.
+ */
+interface SerializedPrompt {
+    /** Prompt unique ID */
+    id: string;
+    /** Version string (e.g., "1.0.0") */
+    version: string;
+    /** System prompt */
+    system: string;
+    /** User prompt template (Mustache format) */
+    userTemplate: string;
+    /** Additional custom fields from AgentPrompt */
+    customFields?: Record<string, unknown>;
+}
+/** Serialized round result for JSON storage */
+interface SerializedRoundResult {
+    /** Round number (1-indexed) */
+    round: number;
+    /** Completion timestamp (ISO 8601) */
+    completedAt: string;
+    /** Average score from this round */
+    avgScore: number;
+    /** Number of passed tests */
+    passed: number;
+    /** Number of failed tests */
+    failed: number;
+    /** Total number of tests */
+    totalTests: number;
+    /** All suggestions generated */
+    suggestionsGenerated: Suggestion[];
+    /** Suggestions that were approved/applied */
+    suggestionsApproved: Suggestion[];
+    /** Prompt snapshot at start of this round */
+    promptSnapshot: SerializedPrompt;
+    /** Prompt version after applying suggestions */
+    promptVersionAfter: string;
+    /** Cost breakdown */
+    cost: RoundCost;
+    /** Score change from previous round */
+    scoreDelta: number | null;
+}
+/**
+ * Improvement cycle history (JSON file schema v1.1.0).
+ * Includes promptSnapshot per round for rollback support.
+ */
+interface ImprovementHistory {
+    /** Schema version for migration compatibility */
+    schemaVersion: '1.1.0';
+    /** Unique session identifier */
+    sessionId: string;
+    /** Session start timestamp (ISO 8601) */
+    startedAt: string;
+    /** Session completion timestamp (ISO 8601, if completed) */
+    completedAt?: string;
+    /** Initial prompt before any improvements */
+    initialPrompt: SerializedPrompt;
+    /** Current/latest prompt */
+    currentPrompt: SerializedPrompt;
+    /** All completed rounds */
+    rounds: SerializedRoundResult[];
+    /** Reason for termination (if completed) */
+    terminationReason?: string;
+    /** Total accumulated cost in USD */
+    totalCost: number;
+}
+/** History persistence configuration */
+interface HistoryConfig {
+    /** Path to save history JSON */
+    path: string;
+    /** Auto-save after each round (default: true) */
+    autoSave?: boolean;
+}
+/** Configuration for running an improvement cycle */
+interface ImprovementCycleConfig<TInput, TOutput> {
+    /** Factory function to create agent with given prompt */
+    createAgent: (prompt: AgentPrompt<TInput>) => EvalAgent<TInput, TOutput>;
+    /** Starting prompt for improvements */
+    initialPrompt: AgentPrompt<TInput>;
+    /** Test cases to evaluate against */
+    testCases: TestCase<TInput>[];
+    /** Judge for evaluation */
+    judge: Judge;
+    /** Improver for generating suggestions */
+    improver: Improver;
+    /** Termination conditions (OR semantics) */
+    terminateWhen: CycleTerminationCondition[];
+    /** Optional configuration */
+    options?: ImprovementCycleOptions;
+}
+/** Optional configuration for improvement cycle */
+interface ImprovementCycleOptions {
+    /** Options passed to eval suite run */
+    runOptions?: {
+        concurrency?: number;
+        iterations?: number;
+    };
+    /** How to bump version on each improvement */
+    versionBump?: 'major' | 'minor' | 'patch';
+    /** Pricing configuration for cost calculation */
+    pricingConfig?: EvalPricingConfig;
+    /** History persistence settings */
+    history?: HistoryConfig;
+    /** Description for agent (passed to judge) */
+    agentDescription?: string;
+    /** Existing session to resume (preserves session ID and accumulated state) */
+    session?: ImprovementSession;
+}
+/** Final result of an improvement cycle */
+interface ImprovementCycleResult<TInput, TOutput> {
+    /** Final improved prompt */
+    finalPrompt: AgentPrompt<TInput>;
+    /** All completed rounds */
+    rounds: RoundResult[];
+    /** Reason for termination */
+    terminationReason: string;
+    /** Total cost in USD */
+    totalCost: number;
+    /** Saved history (if persistence was enabled) */
+    history?: ImprovementHistory;
+}
+declare function isTargetScoreCondition(condition: CycleTerminationCondition): condition is TargetScoreCondition;
+declare function isMaxRoundsCondition(condition: CycleTerminationCondition): condition is MaxRoundsCondition;
+declare function isNoImprovementCondition(condition: CycleTerminationCondition): condition is NoImprovementCondition;
+declare function isMaxCostCondition(condition: CycleTerminationCondition): condition is MaxCostCondition;
+declare function isCustomCycleCondition(condition: CycleTerminationCondition): condition is CustomCycleCondition;
+declare function isCycleTerminated(result: CycleTerminationResult): result is CycleTerminatedResult;
+
+/**
+ * Options for saving an ImprovementCycleResult as JSON.
+ *
+ * Supports two modes:
+ * - **Auto mode**: Provide `outputDir` and `name` to create a timestamped subdirectory
+ * - **Explicit mode**: Provide `directory` to use an existing directory directly
+ */
+interface SaveCycleJsonOptions {
+    /** Base output directory (creates {name}-{timestamp}/ subdirectory) */
+    outputDir?: string;
+    /** Cycle name (used for folder name with timestamp) */
+    name?: string;
+    /** Use this exact directory path (no timestamp suffix added) */
+    directory?: string;
+    /** Whether to save individual round reports (default: true) */
+    saveRounds?: boolean;
+}
+/**
+ * Saves an ImprovementCycleResult to JSON files.
+ *
+ * Creates a directory containing:
+ * - `cycle-summary.json`: Structured cycle summary
+ * - `round-{n}-report.json`: Individual round reports (if saveRounds=true)
+ *
+ * @example Auto mode (creates timestamped directory)
+ * ```typescript
+ * const dir = saveCycleJson(result, {
+ *   outputDir: './reports',
+ *   name: 'my-agent',
+ * })
+ * // -> ./reports/my-agent-1736691234567/
+ * ```
+ *
+ * @example Explicit mode (uses existing directory)
+ * ```typescript
+ * const dir = saveCycleJson(result, {
+ *   directory: './reports/my-existing-dir',
+ * })
+ * // -> ./reports/my-existing-dir/
+ * ```
+ */
+declare function saveCycleJson<TInput, TOutput>(result: ImprovementCycleResult<TInput, TOutput>, options: SaveCycleJsonOptions): string;
+
+/**
+ * Options for logging an ImprovementCycleResult to console.
+ */
+interface LogCycleOptions {
+    /** Verbosity level for per-round details */
+    verbosity?: LogVerbosity;
+    /** Show per-round details (default: false, summary only) */
+    showRounds?: boolean;
+}
+/**
+ * Logs an ImprovementCycleResult to the console.
+ *
+ * Shows cycle summary including round count, termination reason, total cost,
+ * and score progression. Optionally shows per-round details.
+ *
+ * @param result - The improvement cycle result to log
+ * @param options - Logging options
+ *
+ * @example
+ * ```typescript
+ * import { logCycle } from '@agtlantis/eval'
+ *
+ * const result = await runImprovementCycleAuto(config)
+ * logCycle(result, { verbosity: 'detailed', showRounds: true })
+ * ```
+ */
+declare function logCycle<TInput, TOutput>(result: ImprovementCycleResult<TInput, TOutput>, options?: LogCycleOptions): void;
+
+/**
+ * Options for generating cycle markdown.
+ */
+interface CycleMarkdownOptions {
+    /** Include full per-round details (default: true) */
+    includeRoundDetails?: boolean;
+    /** Show prompt evolution - initial vs final (default: false) */
+    showPromptEvolution?: boolean;
+}
+/**
+ * Converts an ImprovementCycleResult to markdown.
+ *
+ * Generates a comprehensive report including:
+ * - Summary table (rounds, termination, cost, scores)
+ * - Score progression table
+ * - Per-round details (optional)
+ * - Prompt evolution (optional)
+ *
+ * @param result - The improvement cycle result
+ * @param options - Markdown generation options
+ * @returns Markdown string
+ *
+ * @example
+ * ```typescript
+ * import { cycleToMarkdown } from '@agtlantis/eval'
+ *
+ * const result = await runImprovementCycleAuto(config)
+ * const markdown = cycleToMarkdown(result, {
+ *   includeRoundDetails: true,
+ *   showPromptEvolution: true,
+ * })
+ * ```
+ */
+declare function cycleToMarkdown<TInput, TOutput>(result: ImprovementCycleResult<TInput, TOutput>, options?: CycleMarkdownOptions): string;
+/**
+ * Saves an ImprovementCycleResult as markdown.
+ *
+ * @param result - The improvement cycle result
+ * @param filePath - Path to save the markdown file
+ * @param options - Markdown generation options
+ *
+ * @example
+ * ```typescript
+ * import { saveCycleMarkdown } from '@agtlantis/eval'
+ *
+ * const result = await runImprovementCycleAuto(config)
+ * saveCycleMarkdown(result, './reports/cycle-report.md', {
+ *   includeRoundDetails: true,
+ * })
+ * ```
+ */
+declare function saveCycleMarkdown<TInput, TOutput>(result: ImprovementCycleResult<TInput, TOutput>, filePath: string, options?: CycleMarkdownOptions): void;
+
+/**
+ * Generates a unified diff string for a suggestion.
+ *
+ * @example
+ * ```typescript
+ * const diff = suggestionDiff(suggestion)
+ * console.log(diff)
+ * // - Old value here
+ * // + New value here
+ * ```
+ */
+declare function suggestionDiff(suggestion: Suggestion): string;
+/**
+ * Generates a preview of what the suggestion would look like when applied.
+ *
+ * @example
+ * ```typescript
+ * const preview = suggestionPreview(suggestion)
+ * console.log(preview)
+ * ```
+ */
+declare function suggestionPreview(suggestion: Suggestion): string;
+/**
+ * Formats a suggestion as a compact summary string.
+ *
+ * @example
+ * ```typescript
+ * console.log(suggestionSummary(suggestion))
+ * // [HIGH] system_prompt: Improve clarity in instructions
+ * ```
+ */
+declare function suggestionSummary(suggestion: Suggestion): string;
+/**
+ * Options for applying suggestions to a prompt.
+ */
+interface ApplyPromptSuggestionsOptions {
+    /**
+     * Version bump type for semver.
+     * - 'major': 1.0.0 → 2.0.0 (breaking changes)
+     * - 'minor': 1.0.0 → 1.1.0 (new features)
+     * - 'patch': 1.0.0 → 1.0.1 (bug fixes)
+     */
+    bumpVersion?: 'major' | 'minor' | 'patch';
+}
+/**
+ * Result of applying suggestions to a prompt.
+ */
+interface ApplySuggestionsResult<TInput, TOutput = unknown> {
+    /** The updated prompt with suggestions applied */
+    prompt: AgentPrompt<TInput>;
+    /** Number of suggestions that were successfully applied */
+    appliedCount: number;
+    /** Suggestions that could not be applied (currentValue not found) */
+    skipped: Array<{
+        suggestion: Suggestion;
+        reason: string;
+    }>;
+}
+/**
+ * Bumps a semver version string.
+ *
+ * @example
+ * ```typescript
+ * bumpVersion('1.0.0', 'major') // '2.0.0'
+ * bumpVersion('1.0.0', 'minor') // '1.1.0'
+ * bumpVersion('1.0.0', 'patch') // '1.0.1'
+ * bumpVersion('1.2.3', 'minor') // '1.3.0'
+ * ```
+ */
+declare function bumpVersion(version: string, bump: 'major' | 'minor' | 'patch'): string;
+/**
+ * Applies approved suggestions to an AgentPrompt and returns a new prompt.
+ *
+ * This function:
+ * - Only applies suggestions where `approved === true`
+ * - For `system_prompt`: replaces `currentValue` in `prompt.system`
+ * - For `user_prompt`: requires `prompt.userTemplate` field, updates it and regenerates `renderUserPrompt`
+ * - For `parameters`: applies to custom fields in the prompt
+ * - Optionally bumps the version (major/minor/patch)
+ *
+ * **Important behaviors:**
+ * - Only the **first occurrence** of `currentValue` is replaced (not all occurrences)
+ * - Special characters like `$&`, `$1` in `suggestedValue` are preserved as-is (no regex interpretation)
+ *
+ * @example
+ * ```typescript
+ * // Apply approved suggestions with minor version bump
+ * const result = applyPromptSuggestions(
+ *   currentPrompt,
+ *   suggestions.filter(s => s.approved),
+ *   { bumpVersion: 'minor' }
+ * )
+ *
+ * console.log(result.prompt.version) // '1.1.0'
+ * console.log(`Applied ${result.appliedCount} suggestions`)
+ *
+ * if (result.skipped.length > 0) {
+ *   console.warn('Skipped suggestions:', result.skipped)
+ * }
+ * ```
+ *
+ * @throws {EvalError} with code SUGGESTION_APPLY_ERROR if:
+ *   - A `user_prompt` suggestion is applied but prompt lacks `userTemplate` field
+ *   - Version format is invalid when bumpVersion is specified
+ */
+declare function applyPromptSuggestions<TInput, TOutput = unknown>(currentPrompt: AgentPrompt<TInput>, suggestions: Suggestion[], options?: ApplyPromptSuggestionsOptions): ApplySuggestionsResult<TInput, TOutput>;
+
+/**
+ * Creates an LLM-based prompt improver.
+ *
+ * Analyzes test results and suggests improvements to the agent's prompt,
+ * focusing on low-scoring criteria with actionable suggestions.
+ *
+ * @example
+ * ```typescript
+ * import { createImprover, defaultImproverPrompt } from '@agtlantis/eval'
+ * import { createGoogleProvider } from '@agtlantis/core'
+ *
+ * const provider = createGoogleProvider({ apiKey }).withDefaultModel('gemini-2.5-flash')
+ *
+ * const improver = createImprover({
+ *   provider,
+ *   prompt: defaultImproverPrompt,
+ * })
+ *
+ * const { suggestions } = await improver.improve(agent.prompt, evaluatedResults)
+ *
+ * for (const suggestion of suggestions) {
+ *   console.log(suggestionDiff(suggestion))
+ *   suggestion.approved = true
+ * }
+ *
+ * const newPrompt = applyPromptSuggestions(agent.prompt, suggestions)
+ * ```
+ */
+declare function createImprover(config: ImproverConfig): Improver;
+
+/**
+ * Configuration for creating a mock agent.
+ */
+interface MockAgentConfig<TInput, TOutput> {
+    /** Name for the mock agent */
+    name?: string;
+    /** Description for the mock agent */
+    description?: string;
+    /** Response to return from execute() */
+    response?: TOutput;
+    /** Token usage to include in metadata */
+    tokenUsage?: EvalTokenUsage;
+    /** Delay in ms before returning response */
+    delay?: number;
+    /** If true, throw an error instead of returning response */
+    shouldError?: boolean;
+    /** Custom error message when shouldError is true */
+    errorMessage?: string;
+    /** Custom execute function for more control */
+    executeFn?: (input: TInput) => Promise<{
+        result: TOutput;
+        metadata?: {
+            tokenUsage?: EvalTokenUsage;
+        };
+    }>;
+}
+/**
+ * Creates a mock agent for testing purposes.
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const agent = createMockAgent<{ query: string }, { answer: string }>({
+ *   response: { answer: 'Hello!' },
+ * })
+ *
+ * // With delay and token usage
+ * const agent = createMockAgent({
+ *   response: { answer: 'Response' },
+ *   delay: 100,
+ *   tokenUsage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+ * })
+ *
+ * // Error testing
+ * const agent = createMockAgent({
+ *   shouldError: true,
+ *   errorMessage: 'Agent failed',
+ * })
+ * ```
+ */
+declare function createMockAgent<TInput, TOutput>(config?: MockAgentConfig<TInput, TOutput>): EvalAgent<TInput, TOutput>;
+/**
+ * Configuration for creating a mock judge.
+ */
+interface MockJudgeConfig {
+    /** Overall score to return (0-100) */
+    score?: number;
+    /** Whether the evaluation passed */
+    passed?: boolean;
+    /** Verdicts to return */
+    verdicts?: Verdict[];
+    /** Metadata to return (for cost tracking tests) */
+    metadata?: JudgeResult['metadata'];
+    /** If true, throw an error instead of returning result */
+    shouldError?: boolean;
+    /** Custom error message when shouldError is true */
+    errorMessage?: string;
+    /** Custom evaluate function for more control */
+    evaluateFn?: (context: EvalContext) => Promise<JudgeResult>;
+}
+/**
+ * Creates a mock judge for testing purposes.
+ *
+ * @example
+ * ```typescript
+ * // Basic usage - passing test
+ * const judge = createMockJudge({
+ *   score: 85,
+ *   passed: true,
+ * })
+ *
+ * // Custom verdicts
+ * const judge = createMockJudge({
+ *   verdicts: [
+ *     { criterionId: 'accuracy', score: 90, reasoning: 'Good', passed: true },
+ *     { criterionId: 'clarity', score: 80, reasoning: 'Clear', passed: true },
+ *   ],
+ *   score: 85,
+ *   passed: true,
+ * })
+ *
+ * // Failing test
+ * const judge = createMockJudge({
+ *   score: 40,
+ *   passed: false,
+ * })
+ *
+ * // Error testing
+ * const judge = createMockJudge({
+ *   shouldError: true,
+ *   errorMessage: 'Judge failed to evaluate',
+ * })
+ * ```
+ */
+declare function createMockJudge(config?: MockJudgeConfig): Judge;
+/**
+ * Configuration for creating a mock improver.
+ */
+interface MockImproverConfig {
+    /** Suggestions to return */
+    suggestions?: Suggestion[];
+    /** If true, throw an error instead of returning suggestions */
+    shouldError?: boolean;
+    /** Custom error message when shouldError is true */
+    errorMessage?: string;
+    /** Custom improve function for more control */
+    improveFn?: (agentPrompt: AgentPrompt<any>, results: TestResultWithVerdict<any, any>[]) => Promise<ImproveResult>;
+}
+/**
+ * Creates a mock improver for testing purposes.
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const improver = createMockImprover({
+ *   suggestions: [
+ *     {
+ *       type: 'system_prompt',
+ *       priority: 'high',
+ *       currentValue: 'Old prompt',
+ *       suggestedValue: 'New prompt',
+ *       reasoning: 'Better clarity',
+ *       expectedImprovement: '10% improvement',
+ *     },
+ *   ],
+ * })
+ *
+ * // Empty suggestions
+ * const improver = createMockImprover({ suggestions: [] })
+ *
+ * // Error testing
+ * const improver = createMockImprover({
+ *   shouldError: true,
+ *   errorMessage: 'Improver failed',
+ * })
+ * ```
+ */
+declare function createMockImprover(config?: MockImproverConfig): Improver;
+
+type TerminationCondition<TInput = unknown, TOutput = unknown> = MaxTurnsCondition | FieldSetCondition | FieldValueCondition | CustomCondition<TInput, TOutput>;
+interface MaxTurnsCondition {
+    type: 'maxTurns';
+    /** Safety limit - terminates after this many turns */
+    count: number;
+}
+interface FieldsCondition {
+    /** Dot notation for nested access (e.g., "result.recommendation") */
+    fieldPath: string;
+}
+interface FieldSetCondition extends FieldsCondition {
+    type: 'fieldSet';
+}
+interface FieldValueCondition extends FieldsCondition {
+    type: 'fieldValue';
+    expectedValue: unknown;
+}
+interface CustomCondition<TInput = unknown, TOutput = unknown> {
+    type: 'custom';
+    /** Sync or async check function (e.g., for LLM-based conditions) */
+    check: (context: ConversationContext<TInput, TOutput>) => boolean | Promise<boolean>;
+    /** For debugging/logging */
+    description?: string;
+}
+type TerminationType = 'condition' | 'maxTurns' | 'error' | 'exhausted';
+interface ContinueResult {
+    terminated: false;
+    reason: string;
+    terminationType?: never;
+    matchedCondition?: never;
+}
+interface TerminatedResult {
+    terminated: true;
+    terminationType: TerminationType;
+    matchedCondition?: TerminationCondition<unknown, unknown>;
+    reason: string;
+}
+type TerminationCheckResult = ContinueResult | TerminatedResult;
+interface ConversationContext<TInput, TOutput = unknown> {
+    currentTurn: number;
+    history: Array<{
+        turn: number;
+        input: TInput;
+        output: TOutput | undefined;
+        metadata?: AgentMetadata;
+    }>;
+    lastOutput?: TOutput;
+}
+interface FollowUpInput<TInput, TOutput = unknown> {
+    /**
+     * Input for this follow-up turn.
+     * Can be static, dynamic (sync), or async (for AI-generated inputs via aiUser()).
+     */
+    input: TInput | ((context: ConversationContext<TInput, TOutput>) => TInput) | ((context: ConversationContext<TInput, TOutput>) => Promise<TInput>);
+    /** For debugging/reports */
+    description?: string;
+    /**
+     * Repeat count (default: 1).
+     * Use Infinity to repeat until termination (must be last followUpInput).
+     */
+    turns?: number;
+}
+interface MultiTurnTestCase<TInput, TOutput = unknown> extends TestCase<TInput> {
+    multiTurn: {
+        /** Inputs for 2nd turn onwards (first turn uses TestCase.input) */
+        followUpInputs?: FollowUpInput<TInput, TOutput>[];
+        /** Any condition triggers termination (OR logic) */
+        terminateWhen: TerminationCondition<TInput, TOutput>[];
+        /** Safety limit (default: 10). Uses min of this and any maxTurns condition. */
+        maxTurns?: number;
+        /** Pass/fail when condition met (default: 'pass') */
+        onConditionMet?: 'pass' | 'fail';
+        /** Pass/fail when maxTurns reached (default: 'fail') */
+        onMaxTurnsReached?: 'pass' | 'fail';
+    };
+}
+interface MultiTurnTestResult<TInput, TOutput> extends Omit<TestResultWithVerdict<TInput, TOutput>, 'output'> {
+    output: TOutput | undefined;
+    conversationHistory: Array<{
+        turn: number;
+        input: TInput;
+        output: TOutput | undefined;
+        metadata?: AgentMetadata;
+    }>;
+    termination: TerminationCheckResult;
+    totalTurns: number;
+}
+declare function isMaxTurnsCondition<TInput, TOutput>(condition: TerminationCondition<TInput, TOutput>): condition is MaxTurnsCondition;
+declare function isFieldSetCondition<TInput, TOutput>(condition: TerminationCondition<TInput, TOutput>): condition is FieldSetCondition;
+declare function isFieldValueCondition<TInput, TOutput>(condition: TerminationCondition<TInput, TOutput>): condition is FieldValueCondition;
+declare function isCustomCondition<TInput, TOutput>(condition: TerminationCondition<TInput, TOutput>): condition is CustomCondition<TInput, TOutput>;
+declare function isMultiTurnTestCase<TInput, TOutput = unknown>(testCase: TestCase<TInput>): testCase is MultiTurnTestCase<TInput, TOutput>;
+declare function isTerminated(result: TerminationCheckResult): result is TerminatedResult;
+
+/** Access a nested field value using dot notation (e.g., "result.recommendation"). */
+declare function getFieldValue(obj: unknown, fieldPath: string): unknown;
+declare function checkCondition<TInput, TOutput>(condition: TerminationCondition<TInput, TOutput>, context: ConversationContext<TInput, TOutput>): Promise<TerminationCheckResult>;
+/** Check all termination conditions (OR relationship). Returns on first termination. */
+declare function checkTermination<TInput, TOutput>(conditions: TerminationCondition<TInput, TOutput>[], context: ConversationContext<TInput, TOutput>): Promise<TerminationCheckResult>;
+
+interface NaturalLanguageConditionOptions {
+    /** Provider to use for evaluation */
+    provider: Provider;
+    /** Prompt describing the termination criteria (e.g., "Has the user's question been fully answered?") */
+    prompt: string;
+    /** Optional system prompt override */
+    systemPrompt?: string;
+}
+/** LLM-based termination condition. Asks the LLM to evaluate the termination criteria. */
+declare function naturalLanguage<TInput = unknown, TOutput = unknown>(options: NaturalLanguageConditionOptions): CustomCondition<TInput, TOutput>;
+/** Terminates when ALL sub-conditions are met (AND logic). */
+declare function and$1<TInput = unknown, TOutput = unknown>(...conditions: TerminationCondition<TInput, TOutput>[]): CustomCondition<TInput, TOutput>;
+/** Terminates when ANY sub-condition is met (OR logic). Useful for nested composites. */
+declare function or$1<TInput = unknown, TOutput = unknown>(...conditions: TerminationCondition<TInput, TOutput>[]): CustomCondition<TInput, TOutput>;
+/** Inverts another condition (NOT logic). */
+declare function not$1<TInput = unknown, TOutput = unknown>(condition: TerminationCondition<TInput, TOutput>): CustomCondition<TInput, TOutput>;
+/** Terminates after a specified number of turns. Convenience wrapper for use in composites. */
+declare function afterTurns<TInput = unknown, TOutput = unknown>(count: number): CustomCondition<TInput, TOutput>;
+/** Terminates when a field matches a specific value. Convenience wrapper for composites. */
+declare function fieldEquals<TInput = unknown, TOutput = unknown>(fieldPath: string, expectedValue: unknown): CustomCondition<TInput, TOutput>;
+/** Terminates when a field is set (not null/undefined). Convenience wrapper for composites. */
+declare function fieldIsSet<TInput = unknown, TOutput = unknown>(fieldPath: string): CustomCondition<TInput, TOutput>;
+
+interface MultiTurnExecuteContext<TInput, TOutput> {
+    agent: EvalAgent<TInput, TOutput>;
+    judge: Judge;
+    agentDescription: string;
+}
+interface MultiTurnExecuteOptions {
+    signal?: AbortSignal;
+}
+declare function executeMultiTurnTestCase<TInput, TOutput>(testCase: MultiTurnTestCase<TInput, TOutput>, context: MultiTurnExecuteContext<TInput, TOutput>, options?: MultiTurnExecuteOptions): Promise<MultiTurnTestResult<TInput, TOutput>>;
+
+interface AIUserOptions<TInput, TOutput> {
+    /** Provider for generating user responses */
+    provider: Provider;
+    /** System prompt (string or function for dynamic personas). Uses default if not provided. */
+    systemPrompt?: string | ((context: ConversationContext<TInput, TOutput>) => string);
+    /** Custom history formatter. Default: JSON-based "User: {input}\nAssistant: {output}" format. */
+    formatHistory?: (context: ConversationContext<TInput, TOutput>) => string;
+    /** Convert LLM text response to TInput. Has access to full context for structured input building. */
+    buildInput: (llmResponse: string, context: ConversationContext<TInput, TOutput>) => TInput;
+}
+/**
+ * Creates an async function that generates user inputs using an LLM for multi-turn testing.
+ *
+ * @example
+ * ```typescript
+ * aiUser({
+ *   provider: openai,
+ *   systemPrompt: 'You are a friendly customer.',
+ *   buildInput: (response, ctx) => ({ message: response }),
+ * })
+ * ```
+ */
+declare function aiUser<TInput, TOutput>(options: AIUserOptions<TInput, TOutput>): (context: ConversationContext<TInput, TOutput>) => Promise<TInput>;
+
+/**
+ * CLI Configuration Types
+ *
+ * Defines the configuration schema for `agent-eval.config.ts` files.
+ * Use `defineConfig()` helper for type inference and IDE autocompletion.
+ */
+
+/**
+ * LLM provider configuration.
+ * API keys fall back to OPENAI_API_KEY or GOOGLE_API_KEY env vars.
+ */
+interface LLMConfig {
+    /** LLM provider */
+    provider: 'openai' | 'gemini';
+    /** API key (optional - falls back to environment variable) */
+    apiKey?: string;
+    /** Default model to use */
+    defaultModel?: string;
+    /**
+     * OpenAI reasoning effort (o1/o3 models only)
+     * @see https://platform.openai.com/docs/guides/reasoning
+     */
+    reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high';
+    /**
+     * Default response format
+     * @see https://platform.openai.com/docs/guides/structured-outputs
+     */
+    defaultResponseFormat?: {
+        type: 'json_object' | 'text';
+    };
+}
+interface CLIJudgeConfig {
+    /**
+     * LLM configuration for judge.
+     * If not specified, uses the main `llm` config.
+     */
+    llm?: LLMConfig;
+    /**
+     * Evaluation criteria.
+     * Use built-in criteria factories like `accuracy()`, `relevance()`,
+     * or define custom criteria objects.
+     */
+    criteria: Array<Criterion | ValidatorCriterion>;
+    /**
+     * Score threshold for passing (0-100).
+     * @default 70
+     */
+    passThreshold?: number;
+    /**
+     * Custom judge prompt.
+     * If not specified, uses the default judge prompt.
+     */
+    prompt?: JudgePrompt;
+}
+interface CLIImproverConfig {
+    /**
+     * LLM configuration for improver.
+     * If not specified, uses the main `llm` config.
+     */
+    llm?: LLMConfig;
+    /**
+     * Custom improver prompt.
+     * If not specified, uses the default improver prompt.
+     */
+    prompt?: ImproverPrompt;
+}
+interface OutputConfig {
+    /**
+     * Directory for report output.
+     * @default './reports'
+     */
+    dir?: string;
+    /**
+     * Custom filename pattern.
+     * Supports `{timestamp}` placeholder.
+     * @default 'eval-{timestamp}.md'
+     */
+    filename?: string;
+    /**
+     * Include verbose details in console output.
+     * @default false
+     */
+    verbose?: boolean;
+}
+interface RunConfig {
+    /**
+     * Number of concurrent test executions.
+     * @default 1
+     */
+    concurrency?: number;
+    /**
+     * Number of iterations per test case (for statistical analysis).
+     * @default 1
+     */
+    iterations?: number;
+    /**
+     * Stop execution on first test failure.
+     * @default false
+     */
+    stopOnFirstFailure?: boolean;
+}
+interface CLISingleTurnTestCase<TInput> extends TestCase<TInput> {
+    /** Test case must NOT have multiTurn field */
+    multiTurn?: never;
+}
+interface CLIMultiTurnTestCase<TInput, TOutput = unknown> extends TestCase<TInput> {
+    /** Multi-turn configuration */
+    multiTurn: {
+        /**
+         * Inputs for 2nd turn onwards.
+         * First turn uses `input` field.
+         */
+        followUpInputs?: FollowUpInput<TInput, TOutput>[];
+        /**
+         * Termination conditions (OR relationship).
+         * Any one triggers termination.
+         */
+        terminateWhen: TerminationCondition<TInput, TOutput>[];
+        /**
+         * Safety limit: maximum turns.
+         * @default 10
+         */
+        maxTurns?: number;
+        /**
+         * Outcome when termination condition is met.
+         * @default 'pass'
+         */
+        onConditionMet?: 'pass' | 'fail';
+        /**
+         * Outcome when maxTurns is reached.
+         * @default 'fail'
+         */
+        onMaxTurnsReached?: 'pass' | 'fail';
+    };
+}
+type CLITestCase<TInput, TOutput = unknown> = CLISingleTurnTestCase<TInput> | CLIMultiTurnTestCase<TInput, TOutput>;
+/**
+ * Main evaluation configuration for CLI.
+ * @typeParam TInput - Agent input type
+ * @typeParam TOutput - Agent output type
+ */
+interface EvalConfig<TInput = unknown, TOutput = unknown> {
+    /**
+     * Human-readable name for this evaluation.
+     */
+    name?: string;
+    /**
+     * Description of what the agent does.
+     * Used by Judge for evaluation context.
+     */
+    agentDescription?: string;
+    /**
+     * The agent to evaluate.
+     */
+    agent: EvalAgent<TInput, TOutput>;
+    /**
+     * LLM configuration (shared by Judge and Improver unless overridden).
+     */
+    llm: LLMConfig;
+    /**
+     * Judge configuration for evaluating agent outputs.
+     */
+    judge: CLIJudgeConfig;
+    /**
+     * Improver configuration for prompt improvement suggestions.
+     * Optional - if not specified, no improvements are generated.
+     */
+    improver?: CLIImproverConfig;
+    /**
+     * Test cases to run (inline TypeScript definition).
+     * Can mix single-turn and multi-turn test cases.
+     *
+     * Either `testCases` or `include` must be provided.
+     * - Use `testCases` for inline TypeScript test case definitions
+     * - Use `include` for YAML-based test case files
+     */
+    testCases?: CLITestCase<TInput, TOutput>[];
+    /**
+     * Output configuration for reports.
+     */
+    output?: OutputConfig;
+    /**
+     * Run configuration for test execution.
+     */
+    run?: RunConfig;
+    /**
+     * Pricing configuration for cost calculation.
+     * If provided, cost breakdown will be included in test metrics.
+     *
+     * @example
+     * ```typescript
+     * pricing: {
+     *   openai: { 'gpt-4o': { inputPricePerMillion: 2.5, outputPricePerMillion: 10 } },
+     *   fallback: { inputPricePerMillion: 1.0, outputPricePerMillion: 3.0 },
+     * }
+     * ```
+     */
+    pricing?: EvalPricingConfig;
+    /**
+     * Glob patterns to discover YAML eval files.
+     * Required when using YAML-based test cases instead of inline testCases.
+     *
+     * @example
+     * ```typescript
+     * include: ['evals/booking/*.eval.yaml']
+     * ```
+     */
+    include?: string[];
+    /**
+     * Agent registry for YAML file references.
+     * YAML files reference agents by name (e.g., `agent: booking-agent`).
+     *
+     * @example
+     * ```typescript
+     * agents: {
+     *   'booking-agent': bookingAgent,
+     *   'qa-agent': qaAgent,
+     * }
+     * ```
+     */
+    agents?: Record<string, EvalAgent<unknown, unknown>>;
+}
+/** Identity function for type inference and IDE autocompletion. */
+declare function defineConfig<TInput = unknown, TOutput = unknown>(config: EvalConfig<TInput, TOutput>): EvalConfig<TInput, TOutput>;
+
+interface DiscoverOptions {
+    /** Override config include patterns (CLI --include) */
+    include?: string[];
+    /** Base directory for glob patterns (defaults to process.cwd()) */
+    cwd?: string;
+    /** Ignore patterns (default excludes node_modules) */
+    ignore?: string[];
+}
+/** Discover YAML eval files matching glob patterns. CLI patterns override config. */
+declare function discoverEvalFiles(config: Pick<EvalConfig, 'include'>, options?: DiscoverOptions): Promise<string[]>;
+
+/** Terminates when the average score reaches or exceeds threshold. */
+declare function targetScore(threshold: number): TargetScoreCondition;
+/** Terminates after completing the specified number of rounds. */
+declare function maxRounds(count: number): MaxRoundsCondition;
+/** Terminates when score hasn't improved for N consecutive rounds. */
+declare function noImprovement(consecutiveRounds: number, minDelta?: number): NoImprovementCondition;
+/** Terminates when total accumulated cost reaches or exceeds the budget. */
+declare function maxCost(maxUSD: number): MaxCostCondition;
+/** Custom termination condition with arbitrary logic. Supports async checks. */
+declare function customCondition(check: (ctx: CycleContext) => boolean | Promise<boolean>, description?: string): CustomCycleCondition;
+/** All conditions must be met for termination. Short-circuits on first false. */
+declare function and(...conditions: CycleTerminationCondition[]): CustomCycleCondition;
+/** Any condition being met causes termination. Short-circuits on first true. */
+declare function or(...conditions: CycleTerminationCondition[]): CustomCycleCondition;
+/** Invert a condition's result. Terminates when inner condition does NOT terminate. */
+declare function not(condition: CycleTerminationCondition): CustomCycleCondition;
+/** Dispatches to the appropriate check function based on condition type. */
+declare function checkCycleCondition(condition: CycleTerminationCondition, context: CycleContext): Promise<CycleTerminationResult>;
+/** Check all conditions with OR semantics - first match wins. */
+declare function checkCycleTermination(conditions: CycleTerminationCondition[], context: CycleContext): Promise<CycleTerminationResult>;
+
+/**
+ * Run an improvement cycle as an AsyncGenerator for Human-in-the-Loop control.
+ * Yields after each round for decision-making (continue, stop, or rollback).
+ */
+declare function runImprovementCycle<TInput, TOutput>(config: ImprovementCycleConfig<TInput, TOutput>): AsyncGenerator<RoundYield, ImprovementCycleResult<TInput, TOutput>, RoundDecision | undefined>;
+/**
+ * Run improvement cycle with automatic approval of all suggestions.
+ * Continues until a termination condition is met.
+ */
+declare function runImprovementCycleAuto<TInput, TOutput>(config: ImprovementCycleConfig<TInput, TOutput>): Promise<ImprovementCycleResult<TInput, TOutput>>;
+
+/**
+ * Options for random selection.
+ */
+interface RandomOptions {
+    /** Seed for reproducible random selection */
+    seed?: number;
+}
+/**
+ * Immutable collection for managing and selecting test cases.
+ * Provides fluent API for filtering, sampling, and accessing test cases.
+ *
+ * ## Immutability
+ * - All selection methods (`filter`, `first`, `random`, etc.) return **new collections**
+ * - Chaining creates intermediate collections without modifying the original
+ * - Internal array is frozen with `Object.freeze()` to prevent accidental mutation
+ * - `toArray()` returns a **mutable copy** for consumer convenience
+ *
+ * @example
+ * ```typescript
+ * import { TestCaseCollection, createEvalSuite } from '@agtlantis/eval'
+ *
+ * const cases = TestCaseCollection.from([
+ *   { id: 'basic', input: { query: 'Hello' } },
+ *   { id: 'complex', input: { query: 'Explain quantum computing' } },
+ *   { id: 'edge', input: { query: '' } },
+ * ])
+ *
+ * // Development: quick feedback
+ * await suite.run(cases.minimal().toArray())
+ *
+ * // CI: full coverage
+ * await suite.run(cases.all().toArray())
+ *
+ * // Debugging specific case
+ * await suite.run(cases.byIds(['edge']).toArray())
+ *
+ * // Chaining: filter then sample
+ * const filtered = cases.filter(tc => tc.tags?.includes('fast')).random(3).toArray()
+ * ```
+ */
+declare class TestCaseCollection<TInput> {
+    private readonly cases;
+    private constructor();
+    /**
+     * Create a collection from an array of test cases.
+     */
+    static from<T>(cases: TestCase<T>[]): TestCaseCollection<T>;
+    /**
+     * Create an empty collection.
+     */
+    static empty<T>(): TestCaseCollection<T>;
+    /**
+     * Number of test cases in the collection.
+     */
+    get length(): number;
+    /**
+     * Whether the collection is empty.
+     */
+    get isEmpty(): boolean;
+    /**
+     * Returns all test cases.
+     * Returns `this` since the collection is immutable (frozen array).
+     * Useful as explicit starting point in chains.
+     */
+    all(): TestCaseCollection<TInput>;
+    /**
+     * Returns the first N test cases (default: 1).
+     * Useful for cost-controlled testing during development.
+     */
+    minimal(count?: number): TestCaseCollection<TInput>;
+    /**
+     * Returns the first N test cases.
+     */
+    first(count: number): TestCaseCollection<TInput>;
+    /**
+     * Returns the last N test cases (default: 1).
+     * Preserves original order (earlier cases first).
+     */
+    last(count?: number): TestCaseCollection<TInput>;
+    /**
+     * Returns N random test cases.
+     *
+     * @param count - Number of cases to select
+     * @param options - Optional seed for reproducibility
+     *
+     * @example
+     * ```typescript
+     * // Different each time
+     * collection.random(5)
+     *
+     * // Same result with same seed
+     * collection.random(5, { seed: 42 })
+     * ```
+     */
+    random(count: number, options?: RandomOptions): TestCaseCollection<TInput>;
+    /**
+     * Filter test cases by predicate.
+     */
+    filter(predicate: (testCase: TestCase<TInput>) => boolean): TestCaseCollection<TInput>;
+    /**
+     * Find test case by ID.
+     * Returns collection with single case or empty collection.
+     */
+    byId(id: string): TestCaseCollection<TInput>;
+    /**
+     * Find test cases by multiple IDs.
+     * Preserves order of provided IDs (first occurrence).
+     * Skips non-existent IDs. Duplicate IDs in input are deduplicated.
+     *
+     * @example
+     * ```typescript
+     * collection.byIds(['a', 'b', 'a'])  // returns [case-a, case-b] (deduplicated)
+     * collection.byIds(['b', 'a'])       // returns [case-b, case-a] (order preserved)
+     * ```
+     */
+    byIds(ids: string[]): TestCaseCollection<TInput>;
+    /**
+     * Get test case by ID.
+     * Returns undefined if not found.
+     */
+    get(id: string): TestCase<TInput> | undefined;
+    /**
+     * Get test case by index.
+     * Supports negative indices (e.g., -1 for last item).
+     * Returns undefined if index is out of bounds.
+     */
+    at(index: number): TestCase<TInput> | undefined;
+    /**
+     * Convert to array.
+     * Returns a mutable copy of the internal array.
+     */
+    toArray(): TestCase<TInput>[];
+    /**
+     * Iterator support for for...of loops and spread operator.
+     */
+    [Symbol.iterator](): Iterator<TestCase<TInput>>;
+}
+/**
+ * Create a single test case with auto-generated ID if not provided.
+ *
+ * Auto-generated IDs use a global counter: `test-1`, `test-2`, etc.
+ *
+ * @param input - The test case input data
+ * @param id - Optional custom ID (uses auto-generated if omitted)
+ * @returns A TestCase object
+ *
+ * @example
+ * ```typescript
+ * const case1 = testCase({ name: 'Alice' })         // id: 'test-1'
+ * const case2 = testCase({ name: 'Bob' })           // id: 'test-2'
+ * const case3 = testCase({ name: 'Charlie' }, 'custom-id')  // id: 'custom-id'
+ * ```
+ *
+ * @remarks
+ * The global counter increments on every call. For deterministic IDs,
+ * provide an explicit ID or use `testCases()` with a prefix.
+ */
+declare function testCase<TInput>(input: TInput, id?: string): TestCase<TInput>;
+/**
+ * Create multiple test cases from inputs.
+ * Auto-generates IDs with optional prefix.
+ *
+ * @example
+ * ```typescript
+ * const cases = testCases([{ name: 'Alice' }, { name: 'Bob' }], 'greet')
+ * // Results in: [{ id: 'greet-0', input: {...} }, { id: 'greet-1', input: {...} }]
+ * ```
+ */
+declare function testCases<TInput>(inputs: TInput[], prefix?: string): TestCase<TInput>[];
+
+export { type AIUserOptions, type AgentMetadata, type AgentPrompt, type AgentResult, type AggregatedMetrics, type ApplyPromptSuggestionsOptions, type ApplySuggestionsResult, type CLIImproverConfig, type CLIJudgeConfig, type CLIMultiTurnTestCase, type CLISingleTurnTestCase, type CLITestCase, type ComponentMetadata, CompositeReporter, ConsoleReporter, type ConsoleReporterOptions, type ContinueResult, type ConversationContext, type ConversationEntry, type CostBreakdown, type CostSummary, type Criterion, type CriterionOptions, type CustomCondition, type CustomCycleCondition, type CycleContext, type CycleContinueResult, type CycleMarkdownOptions, type CycleTerminatedResult, type CycleTerminationCondition, type CycleTerminationResult, type DiscoverOptions, type EvalAgent, type EvalAgentConfig, type EvalConfig, type EvalContext, EvalError, EvalErrorCode, type EvalErrorOptions, type EvalPricingConfig, type EvalReport, type EvalResultKind, type EvalSuite, type EvalSuiteConfig, type EvalTestResult, type EvalTokenUsage, type ExecuteContext, type FieldSetCondition, type FieldValueCondition, type FieldsCondition, type FileContent, type FileContentMetadata, type FileReporterOptions, type FollowUpInput, type HistoryConfig, type HistoryStorage, type ImproveResult, type ImprovementCycleConfig, type ImprovementCycleOptions, type ImprovementCycleResult, type ImprovementHistory, type ImprovementSession, type Improver, type ImproverConfig, type ImproverContext, type ImproverMetadata, type ImproverPrompt, type IterationData, type IterationStats, JsonReporter, type Judge, type JudgeConfig, type JudgeContext, type JudgeMetadata, type JudgePrompt, type JudgeResult, type LLMConfig, type LogCycleOptions, type LogVerbosity, MarkdownReporter, type MarkdownReporterOptions, type MaxCostCondition, type MaxRoundsCondition, type MaxTurnsCondition, type MetricsResult, type MetricsWithCost, type MockAgentConfig, type MockImproverConfig, type MockJudgeConfig, type MultiTurnData, type MultiTurnExecuteContext, type MultiTurnExecuteOptions, type MultiTurnIteratedResult, type MultiTurnIterationStats, type MultiTurnResult, type MultiTurnTestCase, type MultiTurnTestResult, type NaturalLanguageConditionOptions, type NoImprovementCondition, type OutputConfig, type RandomOptions, type ReportComparison, type ReportMarkdownOptions, type ReportRunnerOptions, type ReportSummary, type Reporter, type RoundCost, type RoundDecision, type RoundResult, type RoundYield, type RunConfig, type RunOptions, type SaveCycleJsonOptions, type SchemaOptions, type SchemaValidationResult, type SerializedPrompt, type SerializedRoundResult, type SessionConfig, type SingleTurnIteratedResult, type SingleTurnResult, type Suggestion, type TargetScoreCondition, type TerminatedResult, type TerminationCheckResult, type TerminationCondition, type TerminationInfo, type TestCase, TestCaseCollection, type TestResult, type TestResultWithCost, type TestResultWithVerdict, type ValidatorCriterion, type ValidatorFn, type Verdict, type ZodIssue, accuracy, addCostsToResults, afterTurns, aggregateIterationResults, aiUser, and$1 as and, applyPromptSuggestions, bumpVersion, calculateAvgPassRate, calculateAvgStdDev, calculateIterationStats, calculateMultiTurnIterationStats, calculateReportCosts, calculateResultCost, checkCondition, checkCycleCondition, checkCycleTermination, checkTermination, compareReports, consistency, createCompositeReporter, createConsoleReporter, createDefaultReporter, createEvalSuite, createImprover, createJsonReporter, createJudge, createMarkdownReporter, createMockAgent, createMockImprover, createMockJudge, createReportRunner, createSession, customCondition, and as cycleAnd, not as cycleNot, or as cycleOr, cycleToMarkdown, defaultHistoryStorage, defineConfig, deserializePrompt, discoverEvalFiles, executeMultiTurnTestCase, executeTestCase, fieldEquals, fieldIsSet, getFieldValue, isCustomCondition, isCustomCycleCondition, isCycleTerminated, isFieldSetCondition, isFieldValueCondition, isIteratedResult, isMaxCostCondition, isMaxRoundsCondition, isMaxTurnsCondition, isMultiTurnResult, isMultiTurnTestCase, isNoImprovementCondition, isSingleTurnResult, isTargetScoreCondition, isTerminated, loadHistory, logCycle, maxCost, maxRounds, naturalLanguage, noImprovement, not$1 as not, or$1 as or, relevance, reportToMarkdown, resumeSession, runImprovementCycle, runImprovementCycleAuto, runWithConcurrency, saveCycleJson, saveCycleMarkdown, saveHistory, saveReportMarkdown, schema, selectRepresentativeResult, serializePrompt, suggestionDiff, suggestionPreview, suggestionSummary, targetScore, testCase, testCases, toEvalAgent };